mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-01 11:02:14 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0d30b623a7 | |||
| 6900e67d32 |
@@ -300,6 +300,9 @@ path = "junit.xml"
|
||||
# negative-path siblings of each family stay in as regression guards.
|
||||
# * rustfs#4843 — over-limit archive entry paths hard-reject the whole
|
||||
# archive even under ignore-errors semantics.
|
||||
# * rustfs#4846 — distributed-lock quorum tests misclassify as timeout
|
||||
# under parallel load (multi-node in-process clusters; natural home is
|
||||
# ci-7's nightly cluster lane).
|
||||
[profile.e2e-full]
|
||||
default-filter = """
|
||||
package(e2e_test)
|
||||
@@ -308,6 +311,7 @@ default-filter = """
|
||||
& !test(/^replication_extension_test::/)
|
||||
& !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
|
||||
& !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
|
||||
& !test(/^reliant::lock::test_distributed_lock_(2_nodes_grpc_read_survives_failed_node|4_nodes_grpc_read_write_quorum_split_with_two_failed_nodes)$/)
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
|
||||
@@ -111,18 +111,7 @@ runs:
|
||||
- name: Setup Rust cache
|
||||
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
|
||||
with:
|
||||
# false is rust-cache's own default. With true, cleanup.ts returns
|
||||
# *before* pruning ~/.cargo/registry/src, and config.ts archives the
|
||||
# whole registry — so every cache carried the unpacked source tree of
|
||||
# every dependency, not just "a few extra crates".
|
||||
#
|
||||
# No coverage is lost: getPackages runs `cargo metadata --all-features`,
|
||||
# a strict superset of any single lane's feature closure, and -sys crates
|
||||
# are explicitly exempted from pruning (their src timestamps would
|
||||
# otherwise trigger rebuilds). Anything pruned is re-unpacked from the
|
||||
# .crate files still in registry/cache, whose mtimes crates.io
|
||||
# normalises, so cargo fingerprints stay valid.
|
||||
cache-all-crates: false
|
||||
cache-all-crates: true
|
||||
cache-on-failure: true
|
||||
shared-key: ${{ inputs.cache-shared-key }}
|
||||
save-if: ${{ inputs.cache-save-if }}
|
||||
|
||||
@@ -49,8 +49,6 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ripgrep
|
||||
run: |
|
||||
|
||||
@@ -80,8 +80,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# cargo-deny compiles nothing, so the full setup composite (apt packages,
|
||||
# protoc, flatc, nextest, rustfmt/clippy) was pure overhead here. It does
|
||||
@@ -100,9 +98,7 @@ jobs:
|
||||
- name: Setup Rust cache
|
||||
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
|
||||
with:
|
||||
# Same reasoning as the setup composite: true archives every
|
||||
# dependency's unpacked source tree.
|
||||
cache-all-crates: false
|
||||
cache-all-crates: true
|
||||
cache-on-failure: true
|
||||
shared-key: rustfs-cargo-deny
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
@@ -123,8 +119,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Report unpinned GitHub Actions
|
||||
run: ./scripts/security/check_workflow_pins.sh --enforce
|
||||
@@ -135,9 +129,6 @@ jobs:
|
||||
- name: Check every job declares a timeout
|
||||
run: ./scripts/security/check_job_timeouts.sh
|
||||
|
||||
- name: Check checkouts clear their credentials
|
||||
run: ./scripts/security/check_persist_credentials.sh
|
||||
|
||||
- name: Check preview release workflow policy
|
||||
run: ./scripts/security/check_preview_release_workflow.sh
|
||||
|
||||
@@ -152,8 +143,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Dependency Review
|
||||
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5
|
||||
@@ -185,8 +174,6 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
|
||||
@@ -99,8 +99,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Determine build strategy
|
||||
id: check
|
||||
@@ -259,7 +257,6 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Rust environment
|
||||
@@ -798,7 +795,6 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Create GitHub Release
|
||||
@@ -856,8 +852,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||
|
||||
@@ -65,14 +65,6 @@ on:
|
||||
- ".github/workflows/docker.yml"
|
||||
- ".github/workflows/audit.yml"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
emit_timings:
|
||||
description: >-
|
||||
Also emit cargo --timings for the ci-dev build and upload it. Used to
|
||||
decide whether sccache is worth adopting (rustfs/backlog#1601 gate).
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -107,33 +99,6 @@ jobs:
|
||||
cache-save-if: 'true'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# rustfs/backlog#1601 gate. sccache can only cache compilation units whose
|
||||
# --emit includes link, so it covers workspace rlibs and nothing else:
|
||||
# clippy is metadata-only, and the ~100 test binaries, the rustfs bin and
|
||||
# every build script invoke the system linker. Before spending a bucket,
|
||||
# credentials and a supply-chain boundary on it, measure how much of the
|
||||
# build is actually rlib codegen.
|
||||
#
|
||||
# Read from the report: workspace lib codegen as a share of the build, and
|
||||
# s3select-query's own rlib as a share. The plan adopts sccache only above
|
||||
# 50% and 25% respectively; if linking dominates instead, the answer is
|
||||
# mold/lld plus split-debuginfo, which is exactly the part sccache cannot
|
||||
# touch. Off by default — this doubles the ci-dev build.
|
||||
- name: Build ci-dev superset (with --timings)
|
||||
if: inputs.emit_timings
|
||||
env:
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: cargo build --workspace --all-targets --timings
|
||||
|
||||
- name: Upload cargo timings report
|
||||
if: inputs.emit_timings
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: cargo-timings-ci-dev
|
||||
path: target/cargo-timings/
|
||||
retention-days: 30
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
@@ -148,21 +113,6 @@ jobs:
|
||||
cargo build --workspace --all-targets
|
||||
cargo build -p rustfs --bins --features e2e-test-hooks
|
||||
|
||||
# Runs before rust-cache's post step, so these are the sizes it is about
|
||||
# to archive. Reported so the cache-all-crates decision stays evidence-led:
|
||||
# registry/src is what that flag prunes, registry/cache is what the pruned
|
||||
# sources are re-unpacked from. See rustfs/backlog#1600.
|
||||
- name: Report cache input sizes
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "### Cache input sizes (ci-dev)"
|
||||
echo '```'
|
||||
du -sh ~/.cargo/registry/src ~/.cargo/registry/cache ~/.cargo/registry/index \
|
||||
~/.cargo/git target 2>/dev/null || true
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2.
|
||||
warm-ci-feat-rio:
|
||||
name: Warm ci-feat-rio
|
||||
|
||||
@@ -79,8 +79,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ripgrep
|
||||
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
||||
@@ -127,8 +125,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Docs-only PRs skip the full code CI, but they are exactly where a
|
||||
# planning-type document could be slipped in (git add -f bypasses
|
||||
|
||||
@@ -95,8 +95,6 @@ jobs:
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Typos check with custom config file
|
||||
uses: crate-ci/typos@37bb98842b0d8c4ffebdb75301a13db0267cef89 # master
|
||||
|
||||
@@ -114,8 +112,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install ripgrep
|
||||
run: sudo apt-get update && sudo apt-get install -y ripgrep
|
||||
@@ -357,8 +353,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -397,8 +391,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -441,8 +433,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -471,8 +461,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -504,8 +492,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -546,8 +532,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -610,8 +594,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Full setup with dependency caching: the smoke-suite step below
|
||||
# compiles the e2e_test crate, which pulls in most of the workspace.
|
||||
@@ -696,8 +678,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -741,8 +721,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Clean up previous test run
|
||||
run: |
|
||||
@@ -797,8 +775,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||
@@ -871,8 +847,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||
|
||||
@@ -22,18 +22,11 @@ on:
|
||||
issue_comment:
|
||||
types: [created, edited]
|
||||
|
||||
# Least privilege at the top, widened per job below. This workflow runs on
|
||||
# pull_request_target and issue_comment, so it holds full secrets on every fork
|
||||
# PR and on any comment anyone writes — the one place in this repository where a
|
||||
# compromised action would be handed a repo-write token. It does not check out
|
||||
# or execute PR code, so there is no pwn-request path today, but the blast
|
||||
# radius should not depend on that staying true.
|
||||
#
|
||||
# contents: write in particular was never used: the signature records are
|
||||
# written to rustfs/cla through the scoped app token created below, and nothing
|
||||
# here writes to this repository's contents.
|
||||
permissions:
|
||||
contents: read
|
||||
contents: write
|
||||
pull-requests: write
|
||||
issues: write
|
||||
checks: write
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
|
||||
@@ -43,9 +36,6 @@ jobs:
|
||||
cancel-closed-pr-runs:
|
||||
name: Cancel Closed PR Runs
|
||||
if: github.event_name == 'pull_request_target' && github.event.action == 'closed'
|
||||
# Echoes one line; the run exists only so the concurrency group cancels the
|
||||
# in-flight run of a closed PR.
|
||||
permissions: {}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
@@ -54,13 +44,6 @@ jobs:
|
||||
|
||||
cla:
|
||||
if: ${{ (github.event_name != 'issue_comment' || github.event.issue.pull_request) && (github.event_name != 'pull_request_target' || github.event.action != 'closed') }}
|
||||
# checks: write reports the merge-queue check run; pull-requests and issues
|
||||
# let cla-bot comment and label. contents stays read — see the note above.
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
issues: write
|
||||
pull-requests: write
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
|
||||
@@ -62,8 +62,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -118,8 +116,6 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
|
||||
@@ -98,7 +98,6 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
# For workflow_run events, checkout the specific commit that triggered the workflow
|
||||
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
|
||||
@@ -306,8 +305,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
|
||||
@@ -64,8 +64,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -126,8 +124,6 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
|
||||
@@ -142,8 +142,6 @@ jobs:
|
||||
TEST_MODE: ${{ matrix.test-mode }}
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Provision Python explicitly rather than trusting the runner image to
|
||||
# ship a working pip (ci-1: a bare python3 without pip is what broke the
|
||||
@@ -363,8 +361,6 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
|
||||
@@ -87,8 +87,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -154,8 +152,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download prebuilt fuzz binaries
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||
@@ -211,8 +207,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download prebuilt fuzz binaries
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||
@@ -260,8 +254,6 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
|
||||
@@ -50,8 +50,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout helm chart repo
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Both inputs reach the shell through env rather than `${{ }}`
|
||||
# interpolation. A git ref name may contain `$(...)` — anything without a
|
||||
@@ -123,8 +121,6 @@ jobs:
|
||||
- name: Checkout helm package repo
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
# persist-credentials-exempt: this checkout's token IS the push credential —
|
||||
# the job git-pushes to rustfs/helm below. Clearing it breaks chart publishing.
|
||||
repository: rustfs/helm
|
||||
token: ${{ secrets.RUSTFS_HELM_PACKAGE }}
|
||||
|
||||
|
||||
@@ -58,8 +58,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
|
||||
@@ -125,8 +125,6 @@ jobs:
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Enable buildx
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
@@ -272,8 +270,6 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
|
||||
@@ -40,10 +40,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
# persist-credentials-exempt: update-flake-lock pushes the branch and opens
|
||||
# the PR. It passes FLAKE_UPDATE_TOKEN to create-pull-request itself rather
|
||||
# than reusing .git/config, but that is unverified — exempt until a
|
||||
# workflow_dispatch run confirms it (rustfs/backlog#1602).
|
||||
|
||||
- name: Install Nix
|
||||
uses: DeterminateSystems/determinate-nix-action@629b284231c2a82554b724e357e47fc6020833c8 # v3
|
||||
|
||||
@@ -71,8 +71,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Nix
|
||||
uses: DeterminateSystems/determinate-nix-action@4eea0b33e3d1f02ecfe37cf16e7204c424009606 # v3.21.0
|
||||
|
||||
@@ -99,8 +99,6 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -150,7 +148,6 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0 # baseline is built from origin/main
|
||||
|
||||
- name: Setup Rust environment
|
||||
@@ -397,8 +394,6 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Asserts that the self-hosted runners are still ephemeral — one job per pod.
|
||||
#
|
||||
# This repository is public and its pull_request jobs run on those runners,
|
||||
# executing the PR's own build.rs, proc-macros and tests. The only thing keeping
|
||||
# that code from reaching a later job is that each ARC pod handles exactly one
|
||||
# job and is then destroyed. That guarantee lives in the ARC scale-set
|
||||
# configuration, outside this repository, where it can be changed without any PR
|
||||
# — so it is asserted here from the outside, against real run data, instead of
|
||||
# being assumed.
|
||||
#
|
||||
# Monthly rather than per-PR: the property changes only when someone
|
||||
# reconfigures the scale set, and the check costs a few dozen API calls.
|
||||
# See docs/ci/runners.md and rustfs/backlog#1602.
|
||||
|
||||
name: Runner Hygiene
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: runner-hygiene
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
check-ephemerality:
|
||||
name: Check runner ephemerality
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Exit 2 (inconclusive / broken) is deliberately not a pass: a window
|
||||
# where every sm-* job was still queued would otherwise look identical to
|
||||
# a clean bill of health.
|
||||
- name: Assert one job per self-hosted runner
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: ./scripts/ci/check_runner_ephemerality.sh 40
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [check-ephemerality]
|
||||
# Same ci-8 mechanism as coverage.yml, audit.yml and the nightly lanes:
|
||||
# scheduled runs file a tracking issue, manual dispatch stays quiet so
|
||||
# debugging never produces a spurious alert.
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -63,8 +63,6 @@ jobs:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
|
||||
Generated
+36
-150
@@ -290,11 +290,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ar_archive_writer"
|
||||
version = "0.5.3"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6"
|
||||
checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348"
|
||||
dependencies = [
|
||||
"object 0.39.1",
|
||||
"object 0.37.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -599,16 +599,6 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "assert-json-diff"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "astral-tokio-tar"
|
||||
version = "0.6.4"
|
||||
@@ -953,32 +943,6 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-kms"
|
||||
version = "1.114.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0b7d906608ee41e7ddea9983577ba82200435644d567d63dc34e822e088b453"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-observability",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.5.0",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.140.0"
|
||||
@@ -1194,23 +1158,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-protocol-test",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"bytes",
|
||||
"h2",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
"indexmap 2.14.0",
|
||||
"pin-project-lite",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tower",
|
||||
@@ -1237,25 +1195,6 @@ dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-protocol-test"
|
||||
version = "0.64.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a"
|
||||
dependencies = [
|
||||
"assert-json-diff",
|
||||
"aws-smithy-runtime-api",
|
||||
"base64-simd",
|
||||
"cbor-diag",
|
||||
"ciborium",
|
||||
"http 0.2.12",
|
||||
"pretty_assertions",
|
||||
"regex-lite",
|
||||
"roxmltree",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-query"
|
||||
version = "0.62.0"
|
||||
@@ -1589,7 +1528,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1608,7 +1547,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1659,7 +1598,7 @@ version = "3.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"darling 0.20.11",
|
||||
"ident_case",
|
||||
"prettyplease",
|
||||
"proc-macro2",
|
||||
@@ -1740,9 +1679,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bytesize"
|
||||
version = "2.6.0"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493"
|
||||
checksum = "3d7c8918969267b2932ffd5655509bbbea0833823058c378876953217f5fc50e"
|
||||
|
||||
[[package]]
|
||||
name = "bytestring"
|
||||
@@ -1819,25 +1758,6 @@ dependencies = [
|
||||
"cipher 0.5.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cbor-diag"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429"
|
||||
dependencies = [
|
||||
"bs58",
|
||||
"chrono",
|
||||
"data-encoding",
|
||||
"half",
|
||||
"nom 7.1.3",
|
||||
"num-bigint",
|
||||
"num-rational",
|
||||
"num-traits",
|
||||
"separator",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.0"
|
||||
@@ -1950,7 +1870,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -1968,9 +1888,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.5"
|
||||
version = "4.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf"
|
||||
checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
@@ -1978,9 +1898,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.5"
|
||||
version = "4.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078"
|
||||
checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
@@ -2408,7 +2328,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2433,11 +2353,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.6"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -3634,7 +3554,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.6",
|
||||
"crypto-common 0.1.7",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3894,7 +3814,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -4339,9 +4259,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.9"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4354,7 +4274,7 @@ version = "1.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
||||
dependencies = [
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -5381,7 +5301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5915,7 +5835,8 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.49"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cty",
|
||||
@@ -6324,7 +6245,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.52"
|
||||
source = "git+https://github.com/xonatius/mimalloc_rust.git?rev=1cdadea43e9c5a0f054b65be21200ce580e4eb13#1cdadea43e9c5a0f054b65be21200ce580e4eb13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
@@ -6746,17 +6668,6 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-rational"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
|
||||
dependencies = [
|
||||
"num-bigint",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
@@ -6821,7 +6732,7 @@ version = "5.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"base64 0.21.7",
|
||||
"chrono",
|
||||
"getrandom 0.2.17",
|
||||
"http 1.5.0",
|
||||
@@ -7946,7 +7857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.13.0",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
@@ -7966,7 +7877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.13.0",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph 0.8.3",
|
||||
@@ -7987,7 +7898,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.13.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8000,7 +7911,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.13.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
@@ -8720,15 +8631,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "roxmltree"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b"
|
||||
dependencies = [
|
||||
"xmlparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rsa"
|
||||
version = "0.9.10"
|
||||
@@ -8822,9 +8724,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh"
|
||||
version = "0.62.5"
|
||||
version = "0.62.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da7c230e0ed9cbeb92fbad6c8848985d6df2a1464c0dc247a021abd666e9005e"
|
||||
checksum = "b8b67b5a0d8068c89dcbe9d95df986af7a851d1f3c604525274c37468e60464f"
|
||||
dependencies = [
|
||||
"aes 0.9.2",
|
||||
"aws-lc-rs",
|
||||
@@ -9609,16 +9511,10 @@ dependencies = [
|
||||
"arc-swap",
|
||||
"argon2",
|
||||
"async-trait",
|
||||
"aws-config",
|
||||
"aws-sdk-kms",
|
||||
"aws-smithy-http-client",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
"base64 0.23.0",
|
||||
"chacha20poly1305",
|
||||
"hex",
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"insta",
|
||||
"jiff",
|
||||
"md-5 0.11.0",
|
||||
@@ -9627,7 +9523,6 @@ dependencies = [
|
||||
"moka",
|
||||
"rand 0.10.2",
|
||||
"reqwest",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-security-governance",
|
||||
"rustfs-utils",
|
||||
"rustify",
|
||||
@@ -9813,7 +9708,6 @@ dependencies = [
|
||||
"hotpath",
|
||||
"jiff",
|
||||
"libc",
|
||||
"log",
|
||||
"metrics",
|
||||
"num_cpus",
|
||||
"nvml-wrapper",
|
||||
@@ -9849,7 +9743,6 @@ dependencies = [
|
||||
"tracing-error",
|
||||
"tracing-opentelemetry",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
@@ -9881,7 +9774,6 @@ dependencies = [
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10716,7 +10608,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.9",
|
||||
"generic-array 0.14.7",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -10769,12 +10661,6 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "separator"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5"
|
||||
|
||||
[[package]]
|
||||
name = "seq-macro"
|
||||
version = "0.3.6"
|
||||
@@ -11712,7 +11598,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
|
||||
+4
-6
@@ -173,7 +173,7 @@ tower-http = { version = "0.7.0" }
|
||||
# Serialization and Data Formats
|
||||
apache-avro = "0.21.0"
|
||||
bytes = { version = "1.12.1" }
|
||||
bytesize = "2.6.0"
|
||||
bytesize = "2.4.2"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.2"
|
||||
@@ -227,7 +227,6 @@ atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.10.1" }
|
||||
aws-credential-types = { version = "1.3.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.114.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.140.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.110.0" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.2.0" }
|
||||
@@ -236,7 +235,7 @@ aws-smithy-types = { version = "1.6.1" }
|
||||
base64 = "0.23.0"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
clap = { version = "4.6.5" }
|
||||
clap = { version = "4.6.4" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8" }
|
||||
@@ -341,15 +340,14 @@ libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.1" }
|
||||
rcgen = { version = "0.14.8", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.62.5" }
|
||||
russh = { version = "0.62.4" }
|
||||
russh-sftp = "2.3.0"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
mimalloc = { version = "0.1.52", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13" }
|
||||
libmimalloc-sys = { version = "0.1.49", git = "https://github.com/xonatius/mimalloc_rust.git", rev = "1cdadea43e9c5a0f054b65be21200ce580e4eb13", features = ["extended"] }
|
||||
mimalloc = "0.1.52"
|
||||
hotpath = { version = "0.22.0", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.48" }
|
||||
|
||||
@@ -230,19 +230,6 @@ pub const ENV_RUSTFS_KMS_ENABLE: &str = "RUSTFS_KMS_ENABLE";
|
||||
/// Default value: false
|
||||
pub const DEFAULT_KMS_ENABLE: bool = false;
|
||||
|
||||
/// Environment variable enabling per-key KMS authorization on the SSE-KMS data path.
|
||||
///
|
||||
/// When enabled, an SSE-KMS write additionally requires `kms:GenerateDataKey` and an
|
||||
/// SSE-KMS read additionally requires `kms:Decrypt` on the resolved key, evaluated as
|
||||
/// the requesting identity. SSE-S3 and SSE-C are unaffected.
|
||||
pub const ENV_RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY: &str = "RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY";
|
||||
|
||||
/// Default per-key KMS authorization mode for the SSE-KMS data path.
|
||||
///
|
||||
/// Off for now so deployments whose identity policies only grant s3 actions keep
|
||||
/// working; the default flips to on in a later release.
|
||||
pub const DEFAULT_KMS_ENFORCE_SSE_KEY_POLICY: bool = false;
|
||||
|
||||
/// Environment variable for server KMS backend.
|
||||
pub const ENV_RUSTFS_KMS_BACKEND: &str = "RUSTFS_KMS_BACKEND";
|
||||
|
||||
|
||||
@@ -26,16 +26,75 @@
|
||||
//! Later batches tracked on backlog#1154: config get/set, info, pools status,
|
||||
//! group lifecycle, import/export IAM.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serial_test::serial;
|
||||
use std::error::Error;
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
type BoxError = Box<dyn Error + Send + Sync>;
|
||||
|
||||
/// Signs and sends an admin HTTP request with the given credential, returning
|
||||
/// status and body. Native `/rustfs/admin/v3` requests and responses are plain
|
||||
/// JSON (the MinIO-compat encryption applies only to `/minio/admin/v3` paths).
|
||||
async fn admin_request(
|
||||
base_url: &str,
|
||||
method: http::Method,
|
||||
path_and_query: &str,
|
||||
body: Option<String>,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(StatusCode, String), BoxError> {
|
||||
let url = format!("{base_url}{path_and_query}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
|
||||
let mut builder = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if body.is_some() {
|
||||
builder = builder.header(CONTENT_TYPE, "application/json");
|
||||
}
|
||||
|
||||
let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
|
||||
let signed = sign_v4(builder.body(Body::empty())?, content_len, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
|
||||
let mut request = local_http_client().request(reqwest_method, &url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.body(body);
|
||||
}
|
||||
let response = request.send().await?;
|
||||
let status = response.status();
|
||||
let text = response.text().await.unwrap_or_default();
|
||||
Ok((status, text))
|
||||
}
|
||||
|
||||
/// Root-credential admin request that must succeed; returns the response body.
|
||||
async fn admin_ok(
|
||||
env: &RustFSTestEnvironment,
|
||||
method: http::Method,
|
||||
path_and_query: &str,
|
||||
body: Option<String>,
|
||||
) -> Result<String, BoxError> {
|
||||
let (status, text) = admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
|
||||
}
|
||||
Ok(text)
|
||||
}
|
||||
|
||||
fn build_s3_client(url: &str, access_key: &str, secret_key: &str) -> Client {
|
||||
let config = Config::builder()
|
||||
|
||||
@@ -24,12 +24,7 @@
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::Client as HttpClient;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs as stdfs;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -80,58 +75,6 @@ pub fn local_http_client() -> HttpClient {
|
||||
.expect("failed to build local reqwest client")
|
||||
}
|
||||
|
||||
/// Signs and sends an admin HTTP request with the given credentials.
|
||||
pub(crate) async fn admin_request(
|
||||
base_url: &str,
|
||||
method: http::Method,
|
||||
path_and_query: &str,
|
||||
body: Option<String>,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{base_url}{path_and_query}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
|
||||
let mut request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if body.is_some() {
|
||||
request = request.header(CONTENT_TYPE, "application/json");
|
||||
}
|
||||
|
||||
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "admin request body is too large")?;
|
||||
let signed = sign_v4(request.body(Body::empty())?, content_length, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let mut request = local_http_client().request(method, &url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.body(body);
|
||||
}
|
||||
let response = request.send().await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
Ok((status, body))
|
||||
}
|
||||
|
||||
/// Sends a root-credential admin request and returns its successful response body.
|
||||
pub(crate) async fn admin_ok(
|
||||
env: &RustFSTestEnvironment,
|
||||
method: http::Method,
|
||||
path_and_query: &str,
|
||||
body: Option<String>,
|
||||
) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let (status, response_body) =
|
||||
admin_request(&env.url, method.clone(), path_and_query, body, &env.access_key, &env.secret_key).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("{method} {path_and_query} failed: {status} {response_body}").into());
|
||||
}
|
||||
Ok(response_body)
|
||||
}
|
||||
|
||||
/// Resolve the RustFS binary relative to the workspace.
|
||||
pub fn rustfs_binary_path() -> PathBuf {
|
||||
rustfs_binary_path_with_features(requested_rustfs_build_features().as_deref())
|
||||
|
||||
@@ -15,7 +15,9 @@
|
||||
|
||||
use super::{grpc_lock_client::GrpcLockClient, grpc_lock_server::spawn_lock_server};
|
||||
use rustfs_lock::client::{LockClient, local::LocalClient};
|
||||
use rustfs_lock::{GlobalLockManager, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey};
|
||||
use rustfs_lock::{
|
||||
GlobalLockManager, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockType, NamespaceLock, ObjectKey,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -33,11 +35,7 @@ struct FailingClient;
|
||||
#[async_trait::async_trait]
|
||||
impl rustfs_lock::LockClient for FailingClient {
|
||||
async fn acquire_lock(&self, _request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<LockResponse> {
|
||||
// Match RemoteClient's transport-failure response so the coordinator can count this node toward quorum loss.
|
||||
Ok(LockResponse::failure(
|
||||
"Remote lock RPC failed: simulated gRPC node failure",
|
||||
Duration::ZERO,
|
||||
))
|
||||
Err(LockError::internal("simulated gRPC node failure"))
|
||||
}
|
||||
|
||||
async fn release(&self, _lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
|
||||
|
||||
@@ -382,6 +382,7 @@ struct ManualTransitionRunReport {
|
||||
skipped_delete_marker: u64,
|
||||
skipped_directory: u64,
|
||||
skipped_replication: u64,
|
||||
skipped_already_transitioned: u64,
|
||||
skipped_already_in_flight: u64,
|
||||
skipped_queue_full: u64,
|
||||
skipped_queue_closed: u64,
|
||||
@@ -407,6 +408,41 @@ fn assert_completed_or_in_flight_partial(state: &str, report: &ManualTransitionR
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_conflict_winner_report(state: &str, report: &ManualTransitionRunReport, expected_objects: u64, context: &str) {
|
||||
assert_completed_or_in_flight_partial(state, report, context);
|
||||
if report.skipped_already_in_flight > 0 {
|
||||
assert!(
|
||||
report.scanned <= expected_objects,
|
||||
"{context}: scanned more objects than the conflict scope contains: {report:#?}"
|
||||
);
|
||||
assert!(
|
||||
report.eligible <= expected_objects,
|
||||
"{context}: marked more objects eligible than the conflict scope contains: {report:#?}"
|
||||
);
|
||||
assert_eq!(
|
||||
report.enqueued + report.skipped_already_in_flight,
|
||||
report.eligible,
|
||||
"{context}: partial in-flight accounting must cover every eligible object: {report:#?}"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(report.scanned, expected_objects, "{context}: {report:#?}");
|
||||
assert_eq!(
|
||||
report.eligible + report.skipped_already_transitioned,
|
||||
expected_objects,
|
||||
"{context}: {report:#?}"
|
||||
);
|
||||
assert_eq!(
|
||||
report.enqueued + report.skipped_already_in_flight,
|
||||
expected_objects,
|
||||
"{context}: {report:#?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
report.transition_completed, report.enqueued,
|
||||
"{context}: winner must wait for all queued transitions: {report:#?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ManualTransitionQueueSnapshot {
|
||||
queue_capacity: u64,
|
||||
@@ -1240,15 +1276,8 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")])
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1318,21 +1347,20 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
|
||||
assert_eq!(conflict.cancel_endpoint, status_endpoint);
|
||||
assert!(!conflict.scope_key.is_empty());
|
||||
|
||||
manual_transition_job_cancel(&hot, cancel_endpoint).await?;
|
||||
|
||||
let terminal = wait_for_manual_transition_job_terminal(&hot, status_endpoint, MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT).await?;
|
||||
assert_eq!(terminal.job_id, job_id);
|
||||
assert_eq!(terminal.status, "cancelled", "terminal conflict winner response: {terminal:#?}");
|
||||
assert!(!terminal.report.dry_run);
|
||||
assert_eq!(terminal.report.bucket, MANUAL_ASYNC_CONFLICT_BUCKET);
|
||||
assert_eq!(terminal.report.prefix, accepted.report.prefix);
|
||||
assert!(terminal.report.cancelled, "terminal conflict winner response: {terminal:#?}");
|
||||
assert_eq!(terminal.report.scanned, 0, "terminal conflict winner response: {terminal:#?}");
|
||||
assert_eq!(terminal.report.enqueued, 0, "terminal conflict winner response: {terminal:#?}");
|
||||
assert_eq!(
|
||||
terminal.report.transition_completed, 0,
|
||||
"terminal conflict winner response: {terminal:#?}"
|
||||
assert_conflict_winner_report(
|
||||
&terminal.status,
|
||||
&terminal.report,
|
||||
MANUAL_ASYNC_CONFLICT_OBJECTS as u64,
|
||||
"terminal conflict winner response",
|
||||
);
|
||||
assert_eq!(terminal.report.dry_run_eligible, 0, "terminal conflict winner response: {terminal:#?}");
|
||||
assert_eq!(terminal.report.transition_failed, 0, "terminal conflict winner response: {terminal:#?}");
|
||||
assert_eq!(terminal.report.tier_failure, 0, "terminal conflict winner response: {terminal:#?}");
|
||||
let after_remote_count = cold_tier_object_count(&cold_client).await?;
|
||||
assert!(after_remote_count >= before_remote_count);
|
||||
assert!(after_remote_count <= before_remote_count + MANUAL_ASYNC_CONFLICT_OBJECTS);
|
||||
|
||||
@@ -12,35 +12,22 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_sts::config::retry::RetryConfig;
|
||||
use aws_sdk_sts::config::{Credentials, Region};
|
||||
use aws_sdk_sts::error::ProvideErrorMetadata;
|
||||
use aws_sdk_sts::operation::RequestId;
|
||||
use aws_sdk_sts::{Client, Config};
|
||||
use aws_smithy_http_client::Builder as SmithyHttpClientBuilder;
|
||||
use bytes::Bytes;
|
||||
use http::header::{AUTHORIZATION, CONTENT_TYPE};
|
||||
use http::{Request, Response};
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::body::Incoming;
|
||||
use hyper::server::conn::http1;
|
||||
use hyper::service::service_fn;
|
||||
use hyper_util::rt::TokioIo;
|
||||
use serde_json::Value;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serial_test::serial;
|
||||
use std::collections::BTreeSet;
|
||||
use std::convert::Infallible;
|
||||
use std::error::Error;
|
||||
use std::sync::Arc;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::{Notify, mpsc};
|
||||
use tokio::task::{JoinHandle, JoinSet};
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
type BoxError = Box<dyn Error + Send + Sync>;
|
||||
type TestResult = Result<(), BoxError>;
|
||||
const OPA_AUTH_TOKEN: &str = "sts-opa-token";
|
||||
|
||||
fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Option<&str>) -> Client {
|
||||
let mut config = Config::builder()
|
||||
@@ -62,14 +49,32 @@ fn sts_client(url: &str, access_key: &str, secret_key: &str, session_token: Opti
|
||||
}
|
||||
|
||||
async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(String, String), BoxError> {
|
||||
let body = admin_ok(
|
||||
env,
|
||||
http::Method::PUT,
|
||||
"/rustfs/admin/v3/add-service-accounts",
|
||||
Some(serde_json::json!({ "targetUser": env.access_key.clone() }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
let response: Value = serde_json::from_str(&body)?;
|
||||
let path = "/rustfs/admin/v3/add-service-accounts";
|
||||
let url = format!("{}{path}", env.url);
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
|
||||
let body = serde_json::json!({ "targetUser": env.access_key.clone() }).to_string();
|
||||
let request = http::Request::builder()
|
||||
.method(http::Method::PUT)
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
|
||||
.body(Body::empty())?;
|
||||
let content_length = i64::try_from(body.len()).map_err(|_| "service account request body is too large")?;
|
||||
let signed = sign_v4(request, content_length, &env.access_key, &env.secret_key, "", "us-east-1");
|
||||
let mut request = local_http_client().put(&url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
let response = request.body(body).send().await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("create service account failed: {status} {body}").into());
|
||||
}
|
||||
|
||||
let response: serde_json::Value = serde_json::from_str(&body)?;
|
||||
let access_key = response["credentials"]["accessKey"]
|
||||
.as_str()
|
||||
.ok_or("service account response should contain credentials.accessKey")?
|
||||
@@ -81,237 +86,28 @@ async fn create_root_service_account(env: &RustFSTestEnvironment) -> Result<(Str
|
||||
Ok((access_key, secret_key))
|
||||
}
|
||||
|
||||
async fn create_user_with_policy(
|
||||
env: &RustFSTestEnvironment,
|
||||
user: &str,
|
||||
secret: &str,
|
||||
policy_name: &str,
|
||||
statements: Value,
|
||||
) -> TestResult {
|
||||
create_user(env, user, secret).await?;
|
||||
admin_ok(
|
||||
env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}"),
|
||||
Some(
|
||||
serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": statements,
|
||||
})
|
||||
.to_string(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
admin_ok(
|
||||
env,
|
||||
http::Method::POST,
|
||||
"/rustfs/admin/v3/idp/builtin/policy/attach",
|
||||
Some(serde_json::json!({ "policies": [policy_name], "user": user }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn create_user(env: &RustFSTestEnvironment, user: &str, secret: &str) -> TestResult {
|
||||
admin_ok(
|
||||
env,
|
||||
http::Method::PUT,
|
||||
&format!("/rustfs/admin/v3/add-user?accessKey={user}"),
|
||||
Some(serde_json::json!({ "secretKey": secret, "status": "enabled" }).to_string()),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_access_denied(client: &Client, context: &str) -> TestResult {
|
||||
async fn assert_chaining_denied(client: &Client, credential_kind: &str) -> TestResult {
|
||||
let error = client
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/test")
|
||||
.role_session_name("sts-query-compat-e2e")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("AssumeRole must be denied");
|
||||
.expect_err("credential chaining must be denied");
|
||||
let service_error = error
|
||||
.as_service_error()
|
||||
.ok_or_else(|| format!("{context} should deserialize as an STS service error: {error:?}"))?;
|
||||
.ok_or_else(|| format!("{credential_kind} denial should deserialize as an STS service error: {error:?}"))?;
|
||||
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(403));
|
||||
assert_eq!(service_error.code(), Some("AccessDenied"));
|
||||
assert_eq!(service_error.message(), Some("Access Denied"));
|
||||
assert!(
|
||||
error.request_id().is_some_and(|request_id| !request_id.is_empty()),
|
||||
"{context} should include a request ID"
|
||||
"{credential_kind} denial should include a request ID"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_opa_request(
|
||||
request: Request<Incoming>,
|
||||
requests: mpsc::UnboundedSender<Value>,
|
||||
validation_started: mpsc::UnboundedSender<()>,
|
||||
validation_mode: OpaValidationMode,
|
||||
expected_authorization: Option<String>,
|
||||
) -> Result<Response<Full<Bytes>>, Infallible> {
|
||||
if let Some(expected_authorization) = expected_authorization
|
||||
&& request.headers().get(AUTHORIZATION).and_then(|value| value.to_str().ok()) != Some(expected_authorization.as_str())
|
||||
{
|
||||
return Ok(Response::builder()
|
||||
.status(401)
|
||||
.body(Full::new(Bytes::new()))
|
||||
.expect("static OPA unauthorized response must be valid"));
|
||||
}
|
||||
|
||||
let body = match request.into_body().collect().await {
|
||||
Ok(body) => body.to_bytes(),
|
||||
Err(error) => {
|
||||
return Ok(Response::builder()
|
||||
.status(400)
|
||||
.body(Full::new(Bytes::from(error.to_string())))
|
||||
.expect("static OPA error response must be valid"));
|
||||
}
|
||||
};
|
||||
|
||||
let payload = if body.is_empty() {
|
||||
None
|
||||
} else {
|
||||
match serde_json::from_slice::<Value>(&body) {
|
||||
Ok(payload) => Some(payload),
|
||||
Err(error) => {
|
||||
return Ok(Response::builder()
|
||||
.status(400)
|
||||
.body(Full::new(Bytes::from(error.to_string())))
|
||||
.expect("static OPA error response must be valid"));
|
||||
}
|
||||
}
|
||||
};
|
||||
if payload.is_none() {
|
||||
let _ = validation_started.send(());
|
||||
if let OpaValidationMode::DelayedUnavailable(release) = validation_mode {
|
||||
release.notified().await;
|
||||
return Ok(Response::builder()
|
||||
.status(503)
|
||||
.body(Full::new(Bytes::new()))
|
||||
.expect("static OPA unavailable response must be valid"));
|
||||
}
|
||||
}
|
||||
let allow = match payload.as_ref().and_then(|value| value.pointer("/input/identity/account")) {
|
||||
Some(Value::String(account)) if account == "opaallow" => payload
|
||||
.as_ref()
|
||||
.and_then(|value| value.pointer("/input/context/deny_only"))
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false),
|
||||
Some(Value::String(account)) if account == "opadeny" => false,
|
||||
None => true,
|
||||
_ => false,
|
||||
};
|
||||
if let Some(payload) = payload {
|
||||
let _ = requests.send(payload);
|
||||
}
|
||||
let body =
|
||||
serde_json::to_vec(&serde_json::json!({ "result": { "allow": allow } })).expect("static OPA response must serialize");
|
||||
Ok(Response::builder()
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(Full::new(Bytes::from(body)))
|
||||
.expect("static OPA response must be valid"))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum OpaValidationMode {
|
||||
Ready,
|
||||
DelayedUnavailable(Arc<Notify>),
|
||||
}
|
||||
|
||||
struct OpaMock {
|
||||
url: String,
|
||||
requests: mpsc::UnboundedReceiver<Value>,
|
||||
validation_started: mpsc::UnboundedReceiver<()>,
|
||||
validation_release: Option<Arc<Notify>>,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl OpaMock {
|
||||
async fn start() -> Result<Self, BoxError> {
|
||||
Self::start_with_mode(OpaValidationMode::Ready, Some(OPA_AUTH_TOKEN)).await
|
||||
}
|
||||
|
||||
async fn start_delayed_unavailable() -> Result<Self, BoxError> {
|
||||
let release = Arc::new(Notify::new());
|
||||
Self::start_with_mode(OpaValidationMode::DelayedUnavailable(release), None).await
|
||||
}
|
||||
|
||||
async fn start_with_mode(validation_mode: OpaValidationMode, auth_token: Option<&str>) -> Result<Self, BoxError> {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let url = format!("http://{}/v1/data/rustfs/authz/allow", listener.local_addr()?);
|
||||
let (requests_tx, requests) = mpsc::unbounded_channel();
|
||||
let (validation_started_tx, validation_started) = mpsc::unbounded_channel();
|
||||
let expected_authorization = auth_token.map(|token| format!("Bearer {token}"));
|
||||
let validation_release = match &validation_mode {
|
||||
OpaValidationMode::Ready => None,
|
||||
OpaValidationMode::DelayedUnavailable(release) => Some(Arc::clone(release)),
|
||||
};
|
||||
let task = tokio::spawn(async move {
|
||||
let mut connections = JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
accepted = listener.accept() => {
|
||||
let Ok((stream, _)) = accepted else { break };
|
||||
let requests = requests_tx.clone();
|
||||
let validation_started = validation_started_tx.clone();
|
||||
let validation_mode = validation_mode.clone();
|
||||
let expected_authorization = expected_authorization.clone();
|
||||
connections.spawn(async move {
|
||||
let handler = service_fn(move |request| {
|
||||
handle_opa_request(
|
||||
request,
|
||||
requests.clone(),
|
||||
validation_started.clone(),
|
||||
validation_mode.clone(),
|
||||
expected_authorization.clone(),
|
||||
)
|
||||
});
|
||||
let _ = http1::Builder::new()
|
||||
.serve_connection(TokioIo::new(stream), handler)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
_ = connections.join_next(), if !connections.is_empty() => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(Self {
|
||||
url,
|
||||
requests,
|
||||
validation_started,
|
||||
validation_release,
|
||||
task,
|
||||
})
|
||||
}
|
||||
|
||||
async fn next_request(&mut self) -> Result<Value, BoxError> {
|
||||
timeout(Duration::from_secs(5), self.requests.recv())
|
||||
.await?
|
||||
.ok_or_else(|| "OPA request channel closed".into())
|
||||
}
|
||||
|
||||
async fn wait_for_validation(&mut self) -> TestResult {
|
||||
timeout(Duration::from_secs(5), self.validation_started.recv())
|
||||
.await?
|
||||
.ok_or_else(|| "OPA validation channel closed".into())
|
||||
}
|
||||
|
||||
fn release_validation(&self) {
|
||||
if let Some(release) = &self.validation_release {
|
||||
release.notify_one();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for OpaMock {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
|
||||
@@ -359,7 +155,7 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
|
||||
"signature rejection should include a request ID"
|
||||
);
|
||||
|
||||
assert_access_denied(
|
||||
assert_chaining_denied(
|
||||
&sts_client(
|
||||
&env.url,
|
||||
temporary.access_key_id(),
|
||||
@@ -371,187 +167,7 @@ async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
|
||||
.await?;
|
||||
|
||||
let (service_access_key, service_secret_key) = create_root_service_account(&env).await?;
|
||||
assert_access_denied(
|
||||
&sts_client(&env.url, &service_access_key, &service_secret_key, None),
|
||||
"service-account denial",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let implicit_user = "stsimplicit";
|
||||
let explicit_allow_user = "stsallow";
|
||||
let explicit_deny_user = "stsdeny";
|
||||
let policyless_user = "stspolicyless";
|
||||
let secret = "stsAuthzSecret123";
|
||||
create_user(&env, policyless_user, secret).await?;
|
||||
assert_access_denied(&sts_client(&env.url, policyless_user, secret, None), "policyless user").await?;
|
||||
create_user_with_policy(
|
||||
&env,
|
||||
implicit_user,
|
||||
secret,
|
||||
"sts-implicit-policy",
|
||||
serde_json::json!([{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListAllMyBuckets"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}]),
|
||||
)
|
||||
.await?;
|
||||
create_user_with_policy(
|
||||
&env,
|
||||
explicit_allow_user,
|
||||
secret,
|
||||
"sts-allow-policy",
|
||||
serde_json::json!([{
|
||||
"Effect": "Allow",
|
||||
"Action": ["sts:AssumeRole"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}]),
|
||||
)
|
||||
.await?;
|
||||
create_user_with_policy(
|
||||
&env,
|
||||
explicit_deny_user,
|
||||
secret,
|
||||
"sts-deny-policy",
|
||||
serde_json::json!([
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["sts:AssumeRole"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
},
|
||||
{
|
||||
"Effect": "Deny",
|
||||
"Action": ["sts:AssumeRole"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}
|
||||
]),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for user in [implicit_user, explicit_allow_user] {
|
||||
let output = sts_client(&env.url, user, secret, None)
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/test")
|
||||
.role_session_name("sts-authz-e2e")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format!("{user} should be allowed to call AssumeRole: {error:?}"))?;
|
||||
let credentials = output
|
||||
.credentials()
|
||||
.ok_or_else(|| format!("{user} AssumeRole response should contain credentials"))?;
|
||||
assert!(!credentials.access_key_id().is_empty());
|
||||
assert!(!credentials.secret_access_key().is_empty());
|
||||
assert!(!credentials.session_token().is_empty());
|
||||
}
|
||||
assert_access_denied(&sts_client(&env.url, explicit_deny_user, secret, None), "explicit sts:AssumeRole Deny").await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sts_assume_role_opa_contract() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let secret = "stsOpaSecret123";
|
||||
create_user_with_policy(
|
||||
&env,
|
||||
"opaallow",
|
||||
secret,
|
||||
"sts-opa-local-deny-policy",
|
||||
serde_json::json!([{
|
||||
"Effect": "Deny",
|
||||
"Action": ["sts:AssumeRole"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}]),
|
||||
)
|
||||
.await?;
|
||||
create_user_with_policy(
|
||||
&env,
|
||||
"opadeny",
|
||||
secret,
|
||||
"sts-opa-local-allow-policy",
|
||||
serde_json::json!([{
|
||||
"Effect": "Allow",
|
||||
"Action": ["sts:AssumeRole"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}]),
|
||||
)
|
||||
.await?;
|
||||
|
||||
sts_client(&env.url, "opaallow", secret, None)
|
||||
.assume_role()
|
||||
.role_arn("arn:aws:iam::123456789012:role/test")
|
||||
.role_session_name("sts-opa-contract")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format!("OPA allow should override the local explicit Deny: {error:?}"))?;
|
||||
assert_access_denied(
|
||||
&sts_client(&env.url, "opadeny", secret, None),
|
||||
"OPA denial despite local sts:AssumeRole Allow",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut accounts = BTreeSet::new();
|
||||
for _ in 0..2 {
|
||||
let request = opa.next_request().await?;
|
||||
assert_eq!(request.pointer("/input/action").and_then(Value::as_str), Some("sts:AssumeRole"));
|
||||
assert_eq!(request.pointer("/input/context/deny_only").and_then(Value::as_bool), Some(true));
|
||||
let account = request
|
||||
.pointer("/input/identity/account")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("OPA input should include identity.account")?;
|
||||
accounts.insert(account.to_owned());
|
||||
}
|
||||
assert_eq!(accounts, BTreeSet::from(["opaallow".to_owned(), "opadeny".to_owned()]));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_sts_assume_role_fails_closed_while_opa_is_unavailable() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut opa = OpaMock::start_delayed_unavailable().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str())])
|
||||
.await?;
|
||||
opa.wait_for_validation().await?;
|
||||
|
||||
let user = "opaunavailable";
|
||||
let secret = "stsOpaUnavailableSecret123";
|
||||
create_user_with_policy(
|
||||
&env,
|
||||
user,
|
||||
secret,
|
||||
"sts-opa-unavailable-local-policy",
|
||||
serde_json::json!([{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListAllMyBuckets"],
|
||||
"Resource": ["arn:aws:s3:::*"],
|
||||
}]),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA initialization").await?;
|
||||
|
||||
opa.release_validation();
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
assert_access_denied(&sts_client(&env.url, user, secret, None), "configured OPA validation failure").await?;
|
||||
assert_chaining_denied(&sts_client(&env.url, &service_access_key, &service_secret_key, None), "service account").await?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
|
||||
@@ -172,9 +172,6 @@ pub mod bucket {
|
||||
}
|
||||
|
||||
pub mod replication {
|
||||
pub use crate::bucket::replication::replication_pool::{
|
||||
DurableMrfBacklogSummary, DurableMrfBucketBacklog, durable_mrf_backlog_summary_snapshot,
|
||||
};
|
||||
pub use crate::bucket::replication::{
|
||||
BucketReplicationResyncStatus, BucketStats, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool,
|
||||
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, REPLICATE_INCOMING_DELETE, ReplicateDecision,
|
||||
@@ -422,15 +419,15 @@ pub mod rio {
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS,
|
||||
PeerRestClient, PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
ScannerBucketListing, ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
|
||||
PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
|
||||
ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -46,11 +46,7 @@ use super::replication_target_boundary::{ReplicationTargetStore, replication_obj
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use rustfs_utils::http::{SUFFIX_REPLICATION_TIMESTAMP, get_str};
|
||||
use std::collections::HashMap;
|
||||
use std::collections::hash_map::Entry;
|
||||
use std::sync::Arc;
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::RwLock as StdRwLock;
|
||||
use std::sync::atomic::AtomicI32;
|
||||
use std::sync::atomic::Ordering;
|
||||
use time::OffsetDateTime;
|
||||
@@ -78,66 +74,6 @@ pub struct DurableMrfBacklog {
|
||||
pub entries: Vec<MrfReplicateEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct DurableMrfBucketBacklog {
|
||||
pub bucket: String,
|
||||
pub count: u64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct DurableMrfBacklogSummary {
|
||||
pub available: bool,
|
||||
pub buckets: Vec<DurableMrfBucketBacklog>,
|
||||
}
|
||||
|
||||
static DURABLE_MRF_BACKLOG_SUMMARY: LazyLock<StdRwLock<DurableMrfBacklogSummary>> =
|
||||
LazyLock::new(|| StdRwLock::new(DurableMrfBacklogSummary::default()));
|
||||
|
||||
fn durable_mrf_backlog_summary_from_sizes<I>(entries: I) -> DurableMrfBacklogSummary
|
||||
where
|
||||
I: IntoIterator<Item = (String, i64)>,
|
||||
{
|
||||
let mut buckets = HashMap::<String, DurableMrfBucketBacklog>::new();
|
||||
for (bucket_name, entry_size) in entries {
|
||||
let Ok(size) = u64::try_from(entry_size) else {
|
||||
return DurableMrfBacklogSummary::default();
|
||||
};
|
||||
|
||||
let bucket = match buckets.entry(bucket_name) {
|
||||
Entry::Occupied(entry) => entry.into_mut(),
|
||||
Entry::Vacant(entry) => {
|
||||
let bucket = entry.key().clone();
|
||||
entry.insert(DurableMrfBucketBacklog {
|
||||
bucket,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
};
|
||||
bucket.count = bucket.count.saturating_add(1);
|
||||
bucket.bytes = bucket.bytes.saturating_add(size);
|
||||
}
|
||||
|
||||
DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: buckets.into_values().collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn set_durable_mrf_backlog_summary(summary: DurableMrfBacklogSummary) {
|
||||
match DURABLE_MRF_BACKLOG_SUMMARY.write() {
|
||||
Ok(mut guard) => *guard = summary,
|
||||
Err(poisoned) => *poisoned.into_inner() = summary,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn durable_mrf_backlog_summary_snapshot() -> DurableMrfBacklogSummary {
|
||||
match DURABLE_MRF_BACKLOG_SUMMARY.read() {
|
||||
Ok(guard) => guard.clone(),
|
||||
Err(poisoned) => poisoned.into_inner().clone(),
|
||||
}
|
||||
}
|
||||
|
||||
fn durable_mrf_backlog_from_read(result: Result<Vec<u8>, EcstoreError>) -> DurableMrfBacklog {
|
||||
match result {
|
||||
Ok(data) => match decode_mrf_file(&data) {
|
||||
@@ -297,7 +233,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
let active_counter = self.active_lrg_workers.clone();
|
||||
let storage = self.storage.clone();
|
||||
let stats = self.stats.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut rx = rx;
|
||||
@@ -306,18 +241,10 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
replicate_object(*obj_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
replicate_delete(*del_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -383,22 +310,23 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
// Perform actual replication (placeholder)
|
||||
replicate_object(*obj_info, storage.clone()).await;
|
||||
stats
|
||||
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
// Perform actual replication (placeholder)
|
||||
replicate_object(obj_info.as_ref().clone(), storage.clone()).await;
|
||||
|
||||
stats
|
||||
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
stats.inc_q(&del_info.bucket, 0, true, del_info.op_type).await;
|
||||
// Perform actual delete replication (placeholder)
|
||||
replicate_delete(*del_info, storage.clone()).await;
|
||||
replicate_delete(del_info.as_ref().clone(), storage.clone()).await;
|
||||
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
stats.dec_q(&del_info.bucket, 0, true, del_info.op_type).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -451,18 +379,16 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
active_counter.fetch_add(1, Ordering::SeqCst);
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
replicate_object(*obj_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
stats
|
||||
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
replicate_object(obj_info.as_ref().clone(), storage.clone()).await;
|
||||
stats
|
||||
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
replicate_delete(*del_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
active_counter.fetch_sub(1, Ordering::SeqCst);
|
||||
@@ -603,13 +529,9 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
if !lrg_workers.is_empty() {
|
||||
let index = (hash as usize) % lrg_workers.len();
|
||||
|
||||
if let Some(worker) = lrg_workers.get(index) {
|
||||
self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
if worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
}
|
||||
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
|
||||
if let Some(worker) = lrg_workers.get(index)
|
||||
&& worker.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_err()
|
||||
{
|
||||
// Try to add more workers if possible
|
||||
let max_l_workers = *self.max_l_workers.read().await;
|
||||
let existing = lrg_workers.len();
|
||||
@@ -617,13 +539,17 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
drop(lrg_workers);
|
||||
|
||||
// Queue to MRF if worker is busy.
|
||||
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "large_object").await;
|
||||
let admission =
|
||||
queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "large_object")
|
||||
.await;
|
||||
|
||||
if let Some(resize) = resize {
|
||||
self.resize_lrg_workers(resize.new_count, resize.existing_count).await;
|
||||
}
|
||||
return admission;
|
||||
}
|
||||
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
}
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
}
|
||||
@@ -636,14 +562,12 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
};
|
||||
|
||||
self.stats.inc_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
if channel.try_send(ReplicationOperation::Object(Box::new(ri.clone()))).is_ok() {
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
}
|
||||
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
|
||||
// Queue to MRF if all workers are busy.
|
||||
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "object").await;
|
||||
let admission = queue_mrf_save_admission(&self.mrf_save_tx, ri.to_mrf_entry(), &ri.bucket, &ri.name, "object").await;
|
||||
|
||||
// Try to scale up workers based on priority
|
||||
self.apply_queue_backpressure("object", true, "Replication queue is backpressured")
|
||||
@@ -662,13 +586,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
};
|
||||
|
||||
self.stats.inc_q(&doi.bucket, 0, true, doi.op_type);
|
||||
if channel.try_send(ReplicationOperation::Delete(Box::new(doi.clone()))).is_ok() {
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
}
|
||||
self.stats.dec_q(&doi.bucket, 0, true, doi.op_type);
|
||||
|
||||
let admission = self.queue_mrf_save_admission(doi.to_mrf_entry(), "delete").await;
|
||||
let admission = queue_mrf_save_admission(
|
||||
&self.mrf_save_tx,
|
||||
doi.to_mrf_entry(),
|
||||
&doi.bucket,
|
||||
&doi.delete_object.object_name,
|
||||
"delete",
|
||||
)
|
||||
.await;
|
||||
|
||||
self.apply_queue_backpressure("delete", false, "Replication delete queue is backpressured")
|
||||
.await;
|
||||
@@ -678,18 +607,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues an MRF save operation
|
||||
async fn queue_mrf_save(&self, entry: MrfReplicateEntry) {
|
||||
let _ = self.queue_mrf_save_admission(entry, "mrf_worker").await;
|
||||
}
|
||||
|
||||
async fn queue_mrf_save_admission(&self, entry: MrfReplicateEntry, queue_type: &'static str) -> ReplicationQueueAdmission {
|
||||
let bucket = entry.bucket.clone();
|
||||
let size = entry.size;
|
||||
let is_delete = matches!(entry.op, MrfOpKind::Delete);
|
||||
let admission = queue_mrf_save_entry(&self.mrf_save_tx, entry, queue_type).await;
|
||||
if admission == ReplicationQueueAdmission::Queued {
|
||||
self.stats.inc_q(&bucket, size, is_delete, ReplicationType::Heal);
|
||||
}
|
||||
admission
|
||||
let _ = queue_mrf_save_admission(&self.mrf_save_tx, entry, "", "", "mrf_worker").await;
|
||||
}
|
||||
|
||||
/// Starts the MRF processor — one-shot at startup.
|
||||
@@ -704,13 +622,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let handle = tokio::spawn(async move {
|
||||
let data = match ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_FILE).await {
|
||||
Ok(d) => d,
|
||||
Err(EcstoreError::ConfigNotFound) => {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
Err(EcstoreError::ConfigNotFound) => return, // no file yet — normal on first start
|
||||
Err(e) => {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -741,9 +653,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return;
|
||||
}
|
||||
};
|
||||
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
|
||||
entries.iter().map(|entry| (entry.bucket.clone(), entry.size)),
|
||||
));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
@@ -856,11 +765,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
error = %e,
|
||||
"Failed to clear MRF recovery file after replay — entries may be replayed again on next restart"
|
||||
);
|
||||
} else {
|
||||
set_durable_mrf_backlog_summary(DurableMrfBacklogSummary {
|
||||
available: true,
|
||||
buckets: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
if queued_count > 0 {
|
||||
@@ -889,7 +793,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return;
|
||||
};
|
||||
let storage = self.storage.clone();
|
||||
let stats = self.stats.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
// The on-disk MRF file is a restart-recovery backstop: entries are
|
||||
@@ -915,7 +818,6 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
entry = rx.recv() => match entry {
|
||||
Some(e) => {
|
||||
if pending.len() >= MRF_PENDING_CAP {
|
||||
dec_mrf_entries(stats.as_ref(), std::slice::from_ref(&e));
|
||||
if !capped {
|
||||
capped = true;
|
||||
warn!(
|
||||
@@ -934,31 +836,20 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
// set, not the absolute length, so a large backlog is
|
||||
// not rewritten on every single add).
|
||||
if pending.len() - flushed_len >= 1000 && flush_mrf_to_disk(&pending, &storage).await {
|
||||
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
|
||||
pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
|
||||
));
|
||||
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
|
||||
flushed_len = pending.len();
|
||||
dirty = false;
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Channel closed (pool shutting down) — final flush.
|
||||
if dirty && flush_mrf_to_disk(&pending, &storage).await {
|
||||
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
|
||||
pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
|
||||
));
|
||||
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
|
||||
if dirty {
|
||||
flush_mrf_to_disk(&pending, &storage).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
},
|
||||
_ = interval.tick() => {
|
||||
if dirty && flush_mrf_to_disk(&pending, &storage).await {
|
||||
set_durable_mrf_backlog_summary(durable_mrf_backlog_summary_from_sizes(
|
||||
pending.iter().map(|entry| (entry.bucket.clone(), entry.size)),
|
||||
));
|
||||
dec_mrf_entries(stats.as_ref(), &pending[flushed_len..]);
|
||||
flushed_len = pending.len();
|
||||
dirty = false;
|
||||
}
|
||||
@@ -981,22 +872,24 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
// Perform actual replication (placeholder)
|
||||
replicate_object(*obj_info, self.storage.clone()).await;
|
||||
stats
|
||||
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
// Perform actual replication (placeholder)
|
||||
replicate_object(obj_info.as_ref().clone(), self.storage.clone()).await;
|
||||
|
||||
stats
|
||||
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
// Perform actual delete replication (placeholder)
|
||||
replicate_delete(*del_info, self.storage.clone()).await;
|
||||
stats.inc_q(&del_info.bucket, 0, true, del_info.op_type).await;
|
||||
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
// Perform actual delete replication (placeholder)
|
||||
replicate_delete(del_info.as_ref().clone(), self.storage.clone()).await;
|
||||
|
||||
stats.dec_q(&del_info.bucket, 0, true, del_info.op_type).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1005,30 +898,16 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Worker function for handling large object replication operations
|
||||
async fn add_large_worker(
|
||||
&self,
|
||||
mut rx: Receiver<ReplicationOperation>,
|
||||
active_counter: Arc<AtomicI32>,
|
||||
stats: Arc<ReplicationStats>,
|
||||
storage: Arc<S>,
|
||||
) {
|
||||
async fn add_large_worker(&self, mut rx: Receiver<ReplicationOperation>, active_counter: Arc<AtomicI32>, storage: Arc<S>) {
|
||||
while let Some(operation) = rx.recv().await {
|
||||
active_counter.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
replicate_object(*obj_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
replicate_delete(*del_info, storage.clone()).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1048,19 +927,18 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let bucket = obj_info.bucket.clone();
|
||||
let size = obj_info.size;
|
||||
let delete_marker = obj_info.delete_marker;
|
||||
let op_type = obj_info.op_type;
|
||||
stats
|
||||
.inc_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
|
||||
replicate_object(obj_info.as_ref().clone(), self.storage.clone()).await;
|
||||
|
||||
stats.dec_q(&bucket, size, delete_marker, op_type);
|
||||
stats
|
||||
.dec_q(&obj_info.bucket, obj_info.size, obj_info.delete_marker, obj_info.op_type)
|
||||
.await;
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
let bucket = del_info.bucket.clone();
|
||||
let op_type = del_info.op_type;
|
||||
replicate_delete(*del_info, self.storage.clone()).await;
|
||||
stats.dec_q(&bucket, 0, true, op_type);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1395,34 +1273,29 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn queue_mrf_save_entry(
|
||||
async fn queue_mrf_save_admission(
|
||||
tx: &Sender<MrfReplicateEntry>,
|
||||
entry: MrfReplicateEntry,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
queue_type: &'static str,
|
||||
) -> ReplicationQueueAdmission {
|
||||
let Err(error) = tx.send(entry).await else {
|
||||
if tx.send(entry).await.is_ok() {
|
||||
return ReplicationQueueAdmission::Queued;
|
||||
};
|
||||
let entry = error.0;
|
||||
}
|
||||
|
||||
warn!(
|
||||
event = EVENT_REPLICATION_MRF_QUEUE_UNAVAILABLE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %entry.bucket,
|
||||
object = %entry.object,
|
||||
bucket = %bucket,
|
||||
object = %object,
|
||||
queue_type = queue_type,
|
||||
"MRF save channel unavailable — replication failure entry could not be persisted for retry"
|
||||
);
|
||||
ReplicationQueueAdmission::Missed
|
||||
}
|
||||
|
||||
fn dec_mrf_entries(stats: &ReplicationStats, entries: &[MrfReplicateEntry]) {
|
||||
for entry in entries {
|
||||
stats.dec_q(&entry.bucket, entry.size, matches!(entry.op, MrfOpKind::Delete), ReplicationType::Heal);
|
||||
}
|
||||
}
|
||||
|
||||
/// Encodes `entries` and overwrites the MRF persistence file.
|
||||
/// Returns `true` on success; on failure logs the error and returns `false`.
|
||||
/// Callers must NOT clear their in-memory buffer on `false` so the next tick
|
||||
@@ -2143,147 +2016,6 @@ mod tests {
|
||||
})
|
||||
}
|
||||
|
||||
async fn current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str) -> (i64, i64) {
|
||||
let stats = pool.stats.get_latest_replication_stats(bucket).await;
|
||||
(stats.replication_stats.q_stat.curr.count, stats.replication_stats.q_stat.curr.bytes)
|
||||
}
|
||||
|
||||
async fn wait_for_current_queue(pool: &ReplicationPool<LoadResyncNodeStore>, bucket: &str, expected: (i64, i64)) {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
if current_queue(pool, bucket).await == expected {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("replication queue should reach the expected state");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_worker_admission_counts_channel_backlog_before_receive() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
pool.workers.write().await.push(tx);
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "admission-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 4096,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(current_queue(&pool, "admission-bucket").await, (1, 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn large_worker_admission_counts_channel_backlog_before_receive() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
pool.lrg_workers.write().await.push(tx);
|
||||
let size = 128 * 1024 * 1024;
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "large-admission-bucket".to_string(),
|
||||
name: "large-object".to_string(),
|
||||
size,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(current_queue(&pool, "large-admission-bucket").await, (1, size));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_admission_counts_channel_backlog_before_receive() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
pool.workers.write().await.push(tx);
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_delete_task(DeletedObjectReplicationInfo {
|
||||
bucket: "delete-admission-bucket".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "deleted-object".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(current_queue(&pool, "delete-admission-bucket").await, (1, 0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_worker_drains_current_backlog_after_processing() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
pool.resize_workers(1, 0).await;
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "regular-drain-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
size: 4096,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
wait_for_current_queue(&pool, "regular-drain-bucket", (0, 0)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn large_worker_drains_current_backlog_after_processing() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
pool.resize_lrg_workers(1, 0).await;
|
||||
let size = 128 * 1024 * 1024;
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "large-drain-bucket".to_string(),
|
||||
name: "large-object".to_string(),
|
||||
size,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
wait_for_current_queue(&pool, "large-drain-bucket", (0, 0)).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_delete_worker_drains_current_backlog_after_processing() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
pool.resize_workers(1, 0).await;
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_delete_task(DeletedObjectReplicationInfo {
|
||||
bucket: "delete-drain-bucket".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "deleted-object".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
wait_for_current_queue(&pool, "delete-drain-bucket", (0, 0)).await;
|
||||
}
|
||||
|
||||
fn load_resync_test_metadata() -> Vec<u8> {
|
||||
let mut status = BucketReplicationResyncStatus::new();
|
||||
status.targets_map.insert(
|
||||
@@ -2569,37 +2301,6 @@ mod tests {
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Missed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_counts_mrf_pending_backlog_when_worker_queue_is_full() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", shared))).await;
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
tx.try_send(ReplicationOperation::Object(Box::new(ReplicateObjectInfo {
|
||||
bucket: "runtime-backlog".to_string(),
|
||||
name: "already-buffered".to_string(),
|
||||
size: 1,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})))
|
||||
.expect("test setup should fill the worker queue");
|
||||
pool.workers.write().await.push(tx);
|
||||
|
||||
let admission = pool
|
||||
.queue_replica_task(ReplicateObjectInfo {
|
||||
bucket: "runtime-backlog".to_string(),
|
||||
name: "fallback-object".to_string(),
|
||||
size: 2048,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
|
||||
assert_eq!(admission, ReplicationQueueAdmission::Queued);
|
||||
let queued = pool.stats.get_latest_replication_stats("runtime-backlog").await;
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.bytes, 2048);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replicate_object_info_from_object_info_preserves_ssec_checksum() {
|
||||
let checksum = bytes::Bytes::from_static(b"ssec-checksum");
|
||||
@@ -2641,7 +2342,7 @@ mod tests {
|
||||
|
||||
tx.try_send(first).expect("first MRF entry should fill the test channel");
|
||||
|
||||
let admission = queue_mrf_save_entry(&tx, second, "test");
|
||||
let admission = queue_mrf_save_admission(&tx, second, "bucket", "second", "test");
|
||||
tokio::pin!(admission);
|
||||
|
||||
assert!(
|
||||
@@ -2986,30 +2687,6 @@ mod tests {
|
||||
assert!(missing_file.entries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_mrf_summary_aggregates_entries_by_bucket_for_obs() {
|
||||
let summary =
|
||||
durable_mrf_backlog_summary_from_sizes([("b1".to_string(), 1024), ("b1".to_string(), 512), ("b2".to_string(), 0)]);
|
||||
|
||||
assert!(summary.available);
|
||||
let buckets = summary
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| (bucket.bucket.clone(), bucket))
|
||||
.collect::<HashMap<_, _>>();
|
||||
assert_eq!(buckets["b1"].count, 2);
|
||||
assert_eq!(buckets["b1"].bytes, 1536);
|
||||
assert_eq!(buckets["b2"].count, 1);
|
||||
assert_eq!(buckets["b2"].bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_mrf_summary_marks_invalid_sizes_unavailable() {
|
||||
let invalid = durable_mrf_backlog_summary_from_sizes([("bucket".to_string(), -1)]);
|
||||
assert!(!invalid.available);
|
||||
assert!(invalid.buckets.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_mrf_snapshot_marks_corrupt_or_invalid_data_unavailable() {
|
||||
let corrupt = durable_mrf_backlog_from_read(Ok(vec![0, 1, 2]));
|
||||
|
||||
@@ -23,8 +23,8 @@ use super::replication_stats_boundary::{
|
||||
};
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio::time::interval;
|
||||
@@ -143,7 +143,7 @@ pub struct ReplicationStats {
|
||||
// Active worker statistics
|
||||
pub workers: Arc<Mutex<ActiveWorkerStat>>,
|
||||
// Queue statistics cache
|
||||
pub q_cache: Arc<StdMutex<QueueCache>>,
|
||||
pub q_cache: Arc<Mutex<QueueCache>>,
|
||||
// Proxy statistics cache
|
||||
pub p_cache: Arc<Mutex<ProxyStatsCache>>,
|
||||
// MRF backlog statistics (simplified)
|
||||
@@ -158,7 +158,7 @@ impl ReplicationStats {
|
||||
Self {
|
||||
sr_stats: Arc::new(SRStats::new()),
|
||||
workers: Arc::new(Mutex::new(ActiveWorkerStat::new())),
|
||||
q_cache: Arc::new(StdMutex::new(QueueCache::new())),
|
||||
q_cache: Arc::new(Mutex::new(QueueCache::new())),
|
||||
p_cache: Arc::new(Mutex::new(ProxyStatsCache::new())),
|
||||
mrf_stats: HashMap::new(),
|
||||
cache: Arc::new(RwLock::new(HashMap::new())),
|
||||
@@ -198,9 +198,8 @@ impl ReplicationStats {
|
||||
let mut interval = interval(Duration::from_secs(2));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Ok(mut cache) = q_cache_clone.lock() {
|
||||
cache.update();
|
||||
}
|
||||
let mut cache = q_cache_clone.lock().await;
|
||||
cache.update();
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -392,13 +391,12 @@ impl ReplicationStats {
|
||||
drop(cache);
|
||||
|
||||
{
|
||||
if let Ok(q_cache) = self.q_cache.lock() {
|
||||
for (bucket, queue_stats) in &q_cache.bucket_stats {
|
||||
let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
|
||||
bucket_stats.q_stat = queue_stats.snapshot();
|
||||
bucket_stats.mark_node_local_provider_available();
|
||||
bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
|
||||
}
|
||||
let q_cache = self.q_cache.lock().await;
|
||||
for (bucket, queue_stats) in &q_cache.bucket_stats {
|
||||
let bucket_stats = result.entry(bucket.clone()).or_insert_with(BucketReplicationStats::new);
|
||||
bucket_stats.q_stat = queue_stats.snapshot();
|
||||
bucket_stats.mark_node_local_provider_available();
|
||||
bucket_stats.queue_scope = ReplicationMetricScope::NodeLocal;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -431,11 +429,8 @@ impl ReplicationStats {
|
||||
let boot_time = SystemTime::UNIX_EPOCH; // simplified implementation
|
||||
let uptime = SystemTime::now().duration_since(boot_time).unwrap_or_default().as_secs() as i64;
|
||||
|
||||
let queued = self
|
||||
.q_cache
|
||||
.lock()
|
||||
.map(|q_cache| q_cache.get_site_stats())
|
||||
.unwrap_or_default();
|
||||
let q_cache = self.q_cache.lock().await;
|
||||
let queued = q_cache.get_site_stats();
|
||||
|
||||
let p_cache = self.p_cache.lock().await;
|
||||
let proxied = p_cache.get_site_stats();
|
||||
@@ -638,9 +633,8 @@ impl ReplicationStats {
|
||||
drop(cache);
|
||||
|
||||
{
|
||||
if let Ok(q_cache) = self.q_cache.lock()
|
||||
&& let Some(queue_stats) = q_cache.bucket_stats.get(bucket)
|
||||
{
|
||||
let q_cache = self.q_cache.lock().await;
|
||||
if let Some(queue_stats) = q_cache.bucket_stats.get(bucket) {
|
||||
replication_stats.q_stat = queue_stats.snapshot();
|
||||
}
|
||||
}
|
||||
@@ -671,17 +665,31 @@ impl ReplicationStats {
|
||||
}
|
||||
|
||||
/// Increase queue statistics
|
||||
pub fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
|
||||
if let Ok(mut q_cache) = self.q_cache.lock() {
|
||||
q_cache.inc(bucket, size);
|
||||
}
|
||||
pub async fn inc_q(&self, bucket: &str, size: i64, _is_delete_repl: bool, _op_type: ReplicationType) {
|
||||
let mut q_cache = self.q_cache.lock().await;
|
||||
let stats = q_cache
|
||||
.bucket_stats
|
||||
.entry(bucket.to_string())
|
||||
.or_insert_with(InQueueMetric::default);
|
||||
stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
|
||||
stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
q_cache.sr_queue_stats.curr.now_bytes.fetch_add(size, Ordering::Relaxed);
|
||||
q_cache.sr_queue_stats.curr.now_count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Decrease queue statistics
|
||||
pub fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
|
||||
if let Ok(mut q_cache) = self.q_cache.lock() {
|
||||
q_cache.dec(bucket, size);
|
||||
}
|
||||
pub async fn dec_q(&self, bucket: &str, size: i64, _is_del_marker: bool, _op_type: ReplicationType) {
|
||||
let mut q_cache = self.q_cache.lock().await;
|
||||
let stats = q_cache
|
||||
.bucket_stats
|
||||
.entry(bucket.to_string())
|
||||
.or_insert_with(InQueueMetric::default);
|
||||
stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
|
||||
stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
|
||||
|
||||
q_cache.sr_queue_stats.curr.now_bytes.fetch_sub(size, Ordering::Relaxed);
|
||||
q_cache.sr_queue_stats.curr.now_count.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Increase proxy metrics
|
||||
@@ -793,14 +801,14 @@ mod tests {
|
||||
async fn latest_stats_include_queue_until_drained() {
|
||||
let stats = ReplicationStats::new();
|
||||
|
||||
stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object);
|
||||
stats.inc_q("queued-bucket", 4096, false, ReplicationType::Object).await;
|
||||
let queued = stats.get_latest_replication_stats("queued-bucket").await;
|
||||
assert!(queued.replication_stats.provider_available);
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.count, 1);
|
||||
assert_eq!(queued.replication_stats.q_stat.curr.bytes, 4096);
|
||||
assert_eq!(queued.replication_stats.queue_scope, ReplicationMetricScope::NodeLocal);
|
||||
|
||||
stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object);
|
||||
stats.dec_q("queued-bucket", 4096, false, ReplicationType::Object).await;
|
||||
let drained = stats.get_latest_replication_stats("queued-bucket").await;
|
||||
assert_eq!(drained.replication_stats.q_stat.curr.count, 0);
|
||||
assert_eq!(drained.replication_stats.q_stat.curr.bytes, 0);
|
||||
@@ -903,7 +911,7 @@ mod tests {
|
||||
for _ in 0..32 {
|
||||
let stats = Arc::clone(&stats);
|
||||
tasks.push(tokio::spawn(async move {
|
||||
stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object);
|
||||
stats.inc_q("concurrent-bucket", 7, false, ReplicationType::Object).await;
|
||||
}));
|
||||
}
|
||||
for task in tasks {
|
||||
|
||||
@@ -44,7 +44,7 @@ pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
pub use internode_data_transport::build_internode_data_transport_from_env;
|
||||
pub(crate) use peer_rest_client::TierConfigReloadOutcome;
|
||||
pub use peer_rest_client::{
|
||||
KMS_SIGNAL_SUBSYSTEM, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, SERVICE_SIGNAL_REFRESH_CONFIG,
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerPeerActivity,
|
||||
};
|
||||
pub(crate) use peer_s3_client::heal_bucket_local_on_disks;
|
||||
|
||||
@@ -45,9 +45,9 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
HealControlRequest, LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest,
|
||||
LoadRebalanceMetaRequest, LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest,
|
||||
LocalStorageInfoRequest, Mss, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, ScannerActivityRequest,
|
||||
ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, SignalServiceResponse, StartDecommissionRequest,
|
||||
StartProfilingRequest, StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest,
|
||||
TierMutationControlResponse, TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
ScannerActivityResponse, ServerInfoRequest, SignalServiceRequest, StartDecommissionRequest, StartProfilingRequest,
|
||||
StopRebalanceRequest, TierMutationAbortRequest, TierMutationCommitRequest, TierMutationControlResponse,
|
||||
TierMutationPeerState, TierMutationPrepareRequest, node_service_client::NodeServiceClient,
|
||||
tier_mutation_control_service_client::TierMutationControlServiceClient,
|
||||
};
|
||||
pub use rustfs_protos::{PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS};
|
||||
@@ -71,12 +71,6 @@ use uuid::Uuid;
|
||||
|
||||
pub const SERVICE_SIGNAL_REFRESH_CONFIG: u64 = 1;
|
||||
pub const SERVICE_SIGNAL_RELOAD_DYNAMIC: u64 = 2;
|
||||
/// Dynamic config subsystem for the cluster-persisted KMS configuration.
|
||||
///
|
||||
/// KMS configuration lives in its own cluster object rather than in the server
|
||||
/// config document, so it is not a `ServerConfig` subsystem; it only shares the
|
||||
/// reload signal transport.
|
||||
pub const KMS_SIGNAL_SUBSYSTEM: &str = "kms";
|
||||
const BACKGROUND_HEAL_STATUS_MAX_MESSAGE_SIZE: usize = 64 * 1024;
|
||||
const HEAL_CONTROL_FINGERPRINT_MAX_SIZE: usize = 256;
|
||||
const HEAL_CONTROL_PAYLOAD_MAX_SIZE: usize = 64 * 1024;
|
||||
@@ -105,13 +99,8 @@ fn decode_bucket_stats_response(response: GetBucketStatsDataResponse) -> Result<
|
||||
}
|
||||
|
||||
fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u32) -> Result<()> {
|
||||
// The version stays pinned to DYNAMIC_CONFIG_PROTOCOL_VERSION rather than
|
||||
// being bumped per subsystem: the comparison is shared, so raising it would
|
||||
// retire peers that already converge scanner and heal config correctly.
|
||||
// Subsystems added after a peer was built are rejected by that peer's own
|
||||
// subsystem allow-list, which surfaces as an explicit failed signal.
|
||||
if sig == SERVICE_SIGNAL_RELOAD_DYNAMIC
|
||||
&& matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS | KMS_SIGNAL_SUBSYSTEM)
|
||||
&& matches!(sub_sys, SCANNER_SUB_SYS | HEAL_SUB_SYS)
|
||||
&& protocol_version < rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION
|
||||
{
|
||||
return Err(Error::other(format!("peer does not support dynamic {sub_sys} config convergence")));
|
||||
@@ -1511,22 +1500,6 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
pub async fn signal_service(&self, sig: u64, sub_sys: &str, dry_run: bool, _exec_at: SystemTime) -> Result<()> {
|
||||
self.signal_service_checked(sig, sub_sys, dry_run).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Report the KMS configuration fingerprint the peer is currently running.
|
||||
///
|
||||
/// Sent as a dry-run reload signal so the peer answers without swapping its
|
||||
/// own configuration. `None` means the peer has no KMS configuration. The
|
||||
/// fingerprint is advisory and feeds cluster status reporting only, so the
|
||||
/// response is not proof-signed.
|
||||
pub async fn kms_config_fingerprint(&self) -> Result<Option<String>> {
|
||||
self.signal_service_checked(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, true)
|
||||
.await
|
||||
.map(|response| response.config_fingerprint)
|
||||
}
|
||||
|
||||
async fn signal_service_checked(&self, sig: u64, sub_sys: &str, dry_run: bool) -> Result<SignalServiceResponse> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
@@ -1547,7 +1520,7 @@ impl PeerRestClient {
|
||||
return Err(Error::other(""));
|
||||
}
|
||||
validate_signal_service_protocol(sig, sub_sys, response.protocol_version)?;
|
||||
Ok(response)
|
||||
Ok(())
|
||||
}
|
||||
.await,
|
||||
)
|
||||
@@ -2374,19 +2347,6 @@ mod tests {
|
||||
.expect("full refresh compatibility is guarded by its scanner preflight");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dynamic_kms_config_requires_versioned_peer_acknowledgement() {
|
||||
let err = validate_signal_service_protocol(SERVICE_SIGNAL_RELOAD_DYNAMIC, KMS_SIGNAL_SUBSYSTEM, 0)
|
||||
.expect_err("an unversioned peer must not claim KMS config convergence");
|
||||
assert!(err.to_string().contains("does not support dynamic"));
|
||||
validate_signal_service_protocol(
|
||||
SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
KMS_SIGNAL_SUBSYSTEM,
|
||||
rustfs_protos::DYNAMIC_CONFIG_PROTOCOL_VERSION,
|
||||
)
|
||||
.expect("a current peer should support dynamic KMS config");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_rest_client_marks_network_like_errors() {
|
||||
assert!(PeerRestClient::is_network_like_error(&Error::other("transport error")));
|
||||
|
||||
@@ -5078,7 +5078,7 @@ impl LocalDisk {
|
||||
Ok((buf, mtime))
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "LocalDisk")]
|
||||
#[hotpath::measure]
|
||||
async fn read_metadata_with_dmtime(&self, file_path: impl AsRef<Path>) -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
|
||||
check_path_length(file_path.as_ref().to_string_lossy().as_ref())?;
|
||||
|
||||
@@ -5121,7 +5121,7 @@ impl LocalDisk {
|
||||
Ok((data, modtime))
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "LocalDisk")]
|
||||
#[hotpath::measure]
|
||||
async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef<Path>, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
|
||||
// TODO: timeout support
|
||||
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?;
|
||||
|
||||
@@ -103,7 +103,7 @@ where
|
||||
/// or `out` is larger than one shard. On error `out`'s contents are
|
||||
/// unspecified but never contain bytes that failed the hash check — the copy
|
||||
/// happens only after verification.
|
||||
#[hotpath::measure(impl_type = "BitrotReader")]
|
||||
#[hotpath::measure]
|
||||
pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
|
||||
let want = out.len();
|
||||
self.begin_read(want)?;
|
||||
@@ -303,7 +303,7 @@ where
|
||||
|
||||
/// Write a (hash+data) block. Returns the number of data bytes written.
|
||||
/// Returns an error if called after a short write or if data exceeds shard_size.
|
||||
#[hotpath::measure(label = "BitrotWriter::write", impl_type = "BitrotWriter")]
|
||||
#[hotpath::measure(label = "BitrotWriter::write")]
|
||||
pub async fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
if buf.is_empty() {
|
||||
return Ok(0);
|
||||
|
||||
@@ -691,7 +691,7 @@ impl<R> ParallelReader<R>
|
||||
where
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
#[hotpath::measure(impl_type = "ParallelReader")]
|
||||
#[hotpath::measure]
|
||||
pub async fn read(&mut self) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>) {
|
||||
// On the reconstruction-verifying GET path, read every live shard reader
|
||||
// in lockstep so all readers advance one block per stripe and stay
|
||||
@@ -1505,7 +1505,7 @@ where
|
||||
}
|
||||
|
||||
impl Erasure {
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub async fn decode<W, R>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
|
||||
@@ -28,11 +28,8 @@ use std::vec;
|
||||
use tokio::io::AsyncRead;
|
||||
use tokio::runtime::RuntimeFlavor;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::{JoinError, JoinHandle};
|
||||
use tracing::error;
|
||||
|
||||
/// Queue-capacity input for encoded blocks awaiting shard writers; it is not a
|
||||
/// per-PUT or process-RSS memory limit.
|
||||
const ENV_RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES: &str = "RUSTFS_ERASURE_ENCODE_MAX_INFLIGHT_BYTES";
|
||||
const ENV_RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS: &str = "RUSTFS_ERASURE_ENCODE_BATCH_BLOCKS";
|
||||
const ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST: &str = "RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST";
|
||||
@@ -90,33 +87,6 @@ fn use_bytesmut_ingest() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST, DEFAULT_RUSTFS_ERASURE_ENCODE_BYTESMUT_INGEST)
|
||||
})
|
||||
}
|
||||
|
||||
/// Keeps the encoder producer scoped to its parent future. Tokio detaches a
|
||||
/// task when its `JoinHandle` is dropped, so the producer must be aborted when
|
||||
/// an upload is cancelled before the encode pipeline finishes.
|
||||
struct AbortOnDropTask<T>(JoinHandle<T>);
|
||||
|
||||
impl<T> AbortOnDropTask<T> {
|
||||
fn new(task: JoinHandle<T>) -> Self {
|
||||
Self(task)
|
||||
}
|
||||
|
||||
async fn abort_and_wait(&mut self) {
|
||||
self.0.abort();
|
||||
let _ = (&mut self.0).await;
|
||||
}
|
||||
|
||||
async fn join(&mut self) -> Result<T, JoinError> {
|
||||
(&mut self.0).await
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Drop for AbortOnDropTask<T> {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
|
||||
/// Read up to `limit` bytes into `buf`'s uninitialized spare capacity, appending after its
|
||||
/// current length, and distinguish a clean EOF from a short read.
|
||||
///
|
||||
@@ -163,65 +133,22 @@ fn queued_block_bytes(block: &[Bytes]) -> usize {
|
||||
block.iter().map(Bytes::len).sum()
|
||||
}
|
||||
|
||||
/// Owns an encoded queue entry's gauge contribution until its consumer takes it.
|
||||
struct QueuedInflightBytes {
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
impl QueuedInflightBytes {
|
||||
fn new(bytes: usize) -> Self {
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(bytes);
|
||||
Self { bytes }
|
||||
async fn drain_queued_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Bytes>>) {
|
||||
while let Some(block) = rx.recv().await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_block_bytes(&block));
|
||||
}
|
||||
|
||||
fn settle(&mut self) {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
if bytes != 0 {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for QueuedInflightBytes {
|
||||
fn drop(&mut self) {
|
||||
self.settle();
|
||||
}
|
||||
}
|
||||
|
||||
/// Couples an encoded block with its queue gauge contribution. Keeping the
|
||||
/// guard in the queue entry also covers Tokio sends that complete through a
|
||||
/// permit after the receiver has closed.
|
||||
struct InflightEntry<T> {
|
||||
entry: T,
|
||||
accounting: QueuedInflightBytes,
|
||||
}
|
||||
|
||||
impl<T> InflightEntry<T> {
|
||||
fn new(entry: T, bytes: usize) -> Self {
|
||||
Self {
|
||||
entry,
|
||||
accounting: QueuedInflightBytes::new(bytes),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_inner(mut self) -> T {
|
||||
self.accounting.settle();
|
||||
self.entry
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_queued<T>(
|
||||
sender: &mpsc::Sender<InflightEntry<T>>,
|
||||
entry: T,
|
||||
bytes: usize,
|
||||
) -> Result<(), mpsc::error::SendError<InflightEntry<T>>> {
|
||||
sender.send(InflightEntry::new(entry, bytes)).await
|
||||
}
|
||||
|
||||
fn queued_batch_bytes(batch: &[Vec<Bytes>]) -> usize {
|
||||
batch.iter().map(|block| queued_block_bytes(block)).sum()
|
||||
}
|
||||
|
||||
async fn drain_queued_batched_inflight_bytes(rx: &mut mpsc::Receiver<Vec<Vec<Bytes>>>) {
|
||||
while let Some(batch) = rx.recv().await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
|
||||
}
|
||||
}
|
||||
|
||||
fn dominant_error_summary_label(summary: &WriteQuorumFailureSummary) -> &'static str {
|
||||
summary.dominant_error_label
|
||||
}
|
||||
@@ -577,7 +504,7 @@ impl Erasure {
|
||||
Ok((reader, total))
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub async fn encode<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
@@ -613,14 +540,13 @@ impl Erasure {
|
||||
));
|
||||
}
|
||||
|
||||
// Bound queued encoded blocks by a queue budget; this does not bound
|
||||
// reader, encoder, writer, allocator, or process-RSS memory.
|
||||
// Bound queued encoded blocks by memory budget to avoid per-request spikes.
|
||||
let expanded_block_bytes = self.shard_size().saturating_mul(self.total_shard_count());
|
||||
let max_inflight_bytes = erasure_encode_max_inflight_bytes();
|
||||
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
|
||||
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Bytes>>>(inflight_blocks);
|
||||
let (tx, mut rx) = mpsc::channel::<Vec<Bytes>>(inflight_blocks);
|
||||
|
||||
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
|
||||
let task = tokio::spawn(async move {
|
||||
let block_size = self.block_size;
|
||||
let mut total = 0;
|
||||
if use_bytesmut_ingest {
|
||||
@@ -641,8 +567,10 @@ impl Erasure {
|
||||
let res = self.clone().encode_block_bytes_mut(encode_buf, n).await?;
|
||||
buf = BytesMut::with_capacity(ingest_capacity);
|
||||
let queued_bytes = queued_block_bytes(&res);
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
|
||||
let send_wait_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
|
||||
if let Err(err) = tx.send(res).await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
|
||||
@@ -670,8 +598,10 @@ impl Erasure {
|
||||
let (res, returned_buf) = self.clone().encode_block(encode_buf, n).await?;
|
||||
buf = returned_buf;
|
||||
let queued_bytes = queued_block_bytes(&res);
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(queued_bytes);
|
||||
let send_wait_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = send_queued(&tx, res, queued_bytes).await {
|
||||
if let Err(err) = tx.send(res).await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
record_internal_stage_if_enabled("erasure_encode_send_wait", send_wait_stage_start);
|
||||
@@ -696,7 +626,7 @@ impl Erasure {
|
||||
}
|
||||
|
||||
Ok((reader, total))
|
||||
}));
|
||||
});
|
||||
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
|
||||
@@ -708,10 +638,11 @@ impl Erasure {
|
||||
break;
|
||||
};
|
||||
record_internal_stage_if_enabled("erasure_encode_recv_wait", recv_wait_stage_start);
|
||||
let block = block.into_inner();
|
||||
if block.is_empty() {
|
||||
break;
|
||||
}
|
||||
let queued_bytes = queued_block_bytes(&block);
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_bytes);
|
||||
let write_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = writers.write(block).await {
|
||||
write_err = Some(err);
|
||||
@@ -721,8 +652,9 @@ impl Erasure {
|
||||
}
|
||||
|
||||
if let Some(err) = write_err {
|
||||
task.abort_and_wait().await;
|
||||
drop(rx);
|
||||
task.abort();
|
||||
let _ = task.await;
|
||||
drain_queued_inflight_bytes(&mut rx).await;
|
||||
let shutdown_stage_start = stage_timer_if_enabled();
|
||||
if let Err(shutdown_err) = writers.shutdown().await {
|
||||
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
|
||||
@@ -731,14 +663,14 @@ impl Erasure {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let (reader, total) = task.join().await??;
|
||||
let (reader, total) = task.await??;
|
||||
let shutdown_stage_start = stage_timer_if_enabled();
|
||||
writers.shutdown().await?;
|
||||
record_internal_stage_if_enabled("erasure_encode_shutdown", shutdown_stage_start);
|
||||
Ok((reader, total))
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub async fn encode_batched<R>(
|
||||
self: Arc<Self>,
|
||||
mut reader: R,
|
||||
@@ -760,9 +692,9 @@ impl Erasure {
|
||||
let inflight_blocks = encode_channel_capacity(expanded_block_bytes, max_inflight_bytes);
|
||||
let batch_blocks = encode_batch_block_count().min(inflight_blocks);
|
||||
let channel_capacity = inflight_blocks.div_ceil(batch_blocks).max(1);
|
||||
let (tx, mut rx) = mpsc::channel::<InflightEntry<Vec<Vec<Bytes>>>>(channel_capacity);
|
||||
let (tx, mut rx) = mpsc::channel::<Vec<Vec<Bytes>>>(channel_capacity);
|
||||
|
||||
let mut task = AbortOnDropTask::new(tokio::spawn(async move {
|
||||
let task = tokio::spawn(async move {
|
||||
let block_size = self.block_size;
|
||||
let mut total = 0;
|
||||
let mut buf = vec![0u8; block_size];
|
||||
@@ -781,8 +713,10 @@ impl Erasure {
|
||||
pending_batch.push(res);
|
||||
|
||||
if pending_batch.len() >= batch_blocks {
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
|
||||
let send_wait_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
|
||||
if let Err(err) = tx.send(pending_batch).await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
|
||||
@@ -808,15 +742,17 @@ impl Erasure {
|
||||
}
|
||||
|
||||
if !pending_batch.is_empty() {
|
||||
rustfs_io_metrics::add_ec_encode_inflight_bytes(pending_batch_bytes);
|
||||
let send_wait_stage_start = stage_timer_if_enabled();
|
||||
if let Err(err) = send_queued(&tx, pending_batch, pending_batch_bytes).await {
|
||||
if let Err(err) = tx.send(pending_batch).await {
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(pending_batch_bytes);
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
record_internal_stage_if_enabled("erasure_encode_batched_send_wait", send_wait_stage_start);
|
||||
}
|
||||
|
||||
Ok((reader, total))
|
||||
}));
|
||||
});
|
||||
|
||||
let mut writers = MultiWriter::new(writers, quorum);
|
||||
let mut write_err = None;
|
||||
@@ -827,7 +763,7 @@ impl Erasure {
|
||||
break;
|
||||
};
|
||||
record_internal_stage_if_enabled("erasure_encode_batched_recv_wait", recv_wait_stage_start);
|
||||
let batch = batch.into_inner();
|
||||
rustfs_io_metrics::remove_ec_encode_inflight_bytes(queued_batch_bytes(&batch));
|
||||
let write_stage_start = stage_timer_if_enabled();
|
||||
for block in batch {
|
||||
if let Err(err) = writers.write(block).await {
|
||||
@@ -842,8 +778,9 @@ impl Erasure {
|
||||
}
|
||||
|
||||
if let Some(err) = write_err {
|
||||
task.abort_and_wait().await;
|
||||
drop(rx);
|
||||
task.abort();
|
||||
let _ = task.await;
|
||||
drain_queued_batched_inflight_bytes(&mut rx).await;
|
||||
let shutdown_stage_start = stage_timer_if_enabled();
|
||||
if let Err(shutdown_err) = writers.shutdown().await {
|
||||
error!("failed to shutdown erasure writers after write error: {:?}", shutdown_err);
|
||||
@@ -852,7 +789,7 @@ impl Erasure {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let (reader, total) = task.join().await??;
|
||||
let (reader, total) = task.await??;
|
||||
let shutdown_stage_start = stage_timer_if_enabled();
|
||||
writers.shutdown().await?;
|
||||
record_internal_stage_if_enabled("erasure_encode_batched_shutdown", shutdown_stage_start);
|
||||
@@ -861,7 +798,7 @@ impl Erasure {
|
||||
|
||||
/// Fast path for small inline objects: skip tokio::spawn + mpsc channel.
|
||||
/// Reads all data, encodes directly, writes shards sequentially.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub async fn encode_inline_small<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
@@ -876,7 +813,7 @@ impl Erasure {
|
||||
|
||||
/// Fast path for single-block non-inline objects: avoids the producer/consumer
|
||||
/// pipeline in `encode()` while keeping the same writer/quorum/shutdown semantics.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub async fn encode_single_block_non_inline<R>(
|
||||
self: Arc<Self>,
|
||||
reader: R,
|
||||
@@ -902,104 +839,7 @@ mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt, ReadBuf};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
struct PendingReader {
|
||||
entered: Option<oneshot::Sender<()>>,
|
||||
dropped: Option<oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
impl PendingReader {
|
||||
fn new() -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>) {
|
||||
let (entered_tx, entered_rx) = oneshot::channel();
|
||||
let (dropped_tx, dropped_rx) = oneshot::channel();
|
||||
(
|
||||
Self {
|
||||
entered: Some(entered_tx),
|
||||
dropped: Some(dropped_tx),
|
||||
},
|
||||
entered_rx,
|
||||
dropped_rx,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for PendingReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if let Some(entered) = self.entered.take() {
|
||||
let _ = entered.send(());
|
||||
}
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PendingReader {
|
||||
fn drop(&mut self) {
|
||||
if let Some(dropped) = self.dropped.take() {
|
||||
let _ = dropped.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BlocksThenPendingReader {
|
||||
blocks_remaining: usize,
|
||||
block: Vec<u8>,
|
||||
blocked: Option<oneshot::Sender<()>>,
|
||||
dropped: Option<oneshot::Sender<()>>,
|
||||
final_block: Option<oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
impl BlocksThenPendingReader {
|
||||
fn new(
|
||||
blocks_remaining: usize,
|
||||
block_size: usize,
|
||||
) -> (Self, oneshot::Receiver<()>, oneshot::Receiver<()>, oneshot::Receiver<()>) {
|
||||
let (blocked_tx, blocked_rx) = oneshot::channel();
|
||||
let (dropped_tx, dropped_rx) = oneshot::channel();
|
||||
let (final_block_tx, final_block_rx) = oneshot::channel();
|
||||
(
|
||||
Self {
|
||||
blocks_remaining,
|
||||
block: vec![0x5a; block_size],
|
||||
blocked: Some(blocked_tx),
|
||||
dropped: Some(dropped_tx),
|
||||
final_block: Some(final_block_tx),
|
||||
},
|
||||
blocked_rx,
|
||||
dropped_rx,
|
||||
final_block_rx,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for BlocksThenPendingReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
if self.blocks_remaining == 0 {
|
||||
if let Some(blocked) = self.blocked.take() {
|
||||
let _ = blocked.send(());
|
||||
}
|
||||
return Poll::Pending;
|
||||
}
|
||||
|
||||
if self.blocks_remaining == 1
|
||||
&& let Some(final_block) = self.final_block.take()
|
||||
{
|
||||
let _ = final_block.send(());
|
||||
}
|
||||
self.blocks_remaining -= 1;
|
||||
buf.put_slice(&self.block);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BlocksThenPendingReader {
|
||||
fn drop(&mut self) {
|
||||
if let Some(dropped) = self.dropped.take() {
|
||||
let _ = dropped.send(());
|
||||
}
|
||||
}
|
||||
}
|
||||
use tokio::io::{AsyncWrite, AsyncWriteExt};
|
||||
|
||||
fn erasure_with_zero_block_size() -> Erasure {
|
||||
let mut erasure = Erasure::default();
|
||||
@@ -1057,54 +897,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct FailAfterReaderBlocksWriter {
|
||||
reader_blocked: oneshot::Receiver<()>,
|
||||
writes: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
impl AsyncWrite for FailAfterReaderBlocksWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
match Pin::new(&mut self.reader_blocked).poll(cx) {
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Ready(_) => {
|
||||
self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Poll::Ready(Err(std::io::Error::other("injected write failure after producer blocks")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
struct StallOnWriteWithSignal {
|
||||
entered: Option<oneshot::Sender<()>>,
|
||||
writes: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
impl AsyncWrite for StallOnWriteWithSignal {
|
||||
fn poll_write(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
self.writes.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
if let Some(entered) = self.entered.take() {
|
||||
let _ = entered.send(());
|
||||
}
|
||||
Poll::Pending
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct ShortWriteWriter;
|
||||
|
||||
@@ -1254,202 +1046,6 @@ mod tests {
|
||||
BitrotWriterWrapper::new(CustomWriter::new_tokio_writer(writer), shard_size, HashAlgorithm::None)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum EncodePipeline {
|
||||
Vec,
|
||||
BytesMut,
|
||||
Batched,
|
||||
}
|
||||
|
||||
async fn aborting_encode_drops_blocked_producer(pipeline: EncodePipeline) {
|
||||
const BLOCK_SIZE: usize = 16;
|
||||
|
||||
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let committed = Arc::new(Mutex::new(Vec::new()));
|
||||
let mut writers = vec![Some(bitrot_writer(DeferredCommitWriter::new(committed.clone()), BLOCK_SIZE))];
|
||||
let (reader, entered, dropped) = PendingReader::new();
|
||||
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
|
||||
|
||||
let encode = match pipeline {
|
||||
EncodePipeline::Vec => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await })
|
||||
}
|
||||
EncodePipeline::BytesMut => {
|
||||
tokio::spawn(async move { erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await })
|
||||
}
|
||||
EncodePipeline::Batched => tokio::spawn(async move { erasure.encode_batched(reader, &mut writers, 1).await }),
|
||||
};
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), entered)
|
||||
.await
|
||||
.expect("producer should enter the blocked reader before cancellation")
|
||||
.expect("blocked reader should signal entry");
|
||||
encode.abort();
|
||||
assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
|
||||
tokio::time::timeout(Duration::from_secs(1), dropped)
|
||||
.await
|
||||
.expect("cancelling encode should drop the producer reader")
|
||||
.expect("blocked reader should signal producer drop");
|
||||
assert!(
|
||||
committed.lock().expect("committed buffer should be lockable").is_empty(),
|
||||
"cancelling before the first encoded block must not make data visible"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
gauge_baseline,
|
||||
"cancelling the encode pipeline must preserve the inflight queue gauge"
|
||||
);
|
||||
}
|
||||
|
||||
async fn writer_error_aborts_blocked_producer(pipeline: EncodePipeline) {
|
||||
const BLOCK_SIZE: usize = 16;
|
||||
|
||||
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let blocks_before_pending = match pipeline {
|
||||
EncodePipeline::Batched => encode_batch_block_count(),
|
||||
EncodePipeline::Vec | EncodePipeline::BytesMut => 1,
|
||||
};
|
||||
let (reader, reader_blocked, reader_dropped, _final_block) =
|
||||
BlocksThenPendingReader::new(blocks_before_pending, BLOCK_SIZE);
|
||||
let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let mut writers = vec![Some(bitrot_writer(
|
||||
FailAfterReaderBlocksWriter {
|
||||
reader_blocked,
|
||||
writes: writes.clone(),
|
||||
},
|
||||
BLOCK_SIZE,
|
||||
))];
|
||||
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
|
||||
|
||||
let result = match pipeline {
|
||||
EncodePipeline::Vec => erasure.encode_with_ingest_mode(reader, &mut writers, 1, false).await,
|
||||
EncodePipeline::BytesMut => erasure.encode_with_ingest_mode(reader, &mut writers, 1, true).await,
|
||||
EncodePipeline::Batched => erasure.encode_batched(reader, &mut writers, 1).await,
|
||||
};
|
||||
|
||||
let err = match result {
|
||||
Ok(_) => panic!("writer quorum failure should fail the encode pipeline"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(err.to_string().contains("Failed to write data"));
|
||||
tokio::time::timeout(Duration::from_secs(1), reader_dropped)
|
||||
.await
|
||||
.expect("writer failure should abort the blocked producer")
|
||||
.expect("blocked producer should signal reader drop");
|
||||
assert_eq!(
|
||||
writes.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"writer failure must stop the pipeline before any additional shard write"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
gauge_baseline,
|
||||
"writer failure must settle all queued and pending encoded bytes"
|
||||
);
|
||||
}
|
||||
|
||||
async fn aborting_full_queue_settles_pending_send() {
|
||||
const BLOCK_SIZE: usize = 16;
|
||||
|
||||
let gauge_baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let erasure = Arc::new(Erasure::new(1, 0, BLOCK_SIZE));
|
||||
let inflight_blocks = encode_channel_capacity(
|
||||
erasure.shard_size().saturating_mul(erasure.total_shard_count()),
|
||||
erasure_encode_max_inflight_bytes(),
|
||||
);
|
||||
let (reader, _reader_blocked, reader_dropped, final_block) =
|
||||
BlocksThenPendingReader::new(inflight_blocks + 2, BLOCK_SIZE);
|
||||
let (writer_entered_tx, writer_entered) = oneshot::channel();
|
||||
let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let mut writers = vec![Some(bitrot_writer(
|
||||
StallOnWriteWithSignal {
|
||||
entered: Some(writer_entered_tx),
|
||||
writes: writes.clone(),
|
||||
},
|
||||
BLOCK_SIZE,
|
||||
))];
|
||||
let erasure_for_task = erasure.clone();
|
||||
let encode = tokio::spawn(async move { erasure_for_task.encode_with_ingest_mode(reader, &mut writers, 1, false).await });
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(1), writer_entered)
|
||||
.await
|
||||
.expect("consumer should start the first writer call")
|
||||
.expect("stalling writer should signal entry");
|
||||
tokio::time::timeout(Duration::from_secs(1), final_block)
|
||||
.await
|
||||
.expect("producer should supply the block whose send fills the queue")
|
||||
.expect("reader should signal final block");
|
||||
|
||||
let expected_queued_bytes =
|
||||
u64::try_from((inflight_blocks + 1) * BLOCK_SIZE).expect("queued byte count should fit the gauge");
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while rustfs_io_metrics::current_ec_encode_inflight_bytes() < gauge_baseline + expected_queued_bytes {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("producer should account for the pending send after the queue fills");
|
||||
|
||||
encode.abort();
|
||||
assert!(matches!(encode.await, Err(err) if err.is_cancelled()), "encode task should be cancelled");
|
||||
tokio::time::timeout(Duration::from_secs(1), reader_dropped)
|
||||
.await
|
||||
.expect("cancelling a full queue should abort its producer")
|
||||
.expect("full-queue producer should signal reader drop");
|
||||
assert_eq!(
|
||||
writes.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"cancellation must not resume the stalled writer"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
gauge_baseline,
|
||||
"cancelling a full queue must settle queued and pending bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelling_vec_encode_drops_blocked_producer() {
|
||||
aborting_encode_drops_blocked_producer(EncodePipeline::Vec).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelling_bytesmut_encode_drops_blocked_producer() {
|
||||
aborting_encode_drops_blocked_producer(EncodePipeline::BytesMut).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelling_batched_encode_drops_blocked_producer() {
|
||||
aborting_encode_drops_blocked_producer(EncodePipeline::Batched).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn vec_writer_error_aborts_blocked_producer() {
|
||||
writer_error_aborts_blocked_producer(EncodePipeline::Vec).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bytesmut_writer_error_aborts_blocked_producer() {
|
||||
writer_error_aborts_blocked_producer(EncodePipeline::BytesMut).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn batched_writer_error_aborts_blocked_producer() {
|
||||
writer_error_aborts_blocked_producer(EncodePipeline::Batched).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cancelling_full_queue_settles_pending_send() {
|
||||
aborting_full_queue_settles_pending_send().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn helper_writers_cover_flush_and_shutdown_paths() {
|
||||
let mut failing_write = FailingWriteWriter;
|
||||
@@ -1733,99 +1329,25 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn queued_inflight_bytes_are_settled_on_all_queue_exit_paths() {
|
||||
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let (tx, rx) = mpsc::channel(1);
|
||||
let queued = vec![Bytes::from_static(b"queued")];
|
||||
let queued_bytes = queued_block_bytes(&queued);
|
||||
async fn drain_queued_inflight_bytes_consumes_pending_blocks() {
|
||||
let (tx, mut rx) = mpsc::channel(2);
|
||||
tx.send(vec![Bytes::from_static(b"queued")]).await.unwrap();
|
||||
drop(tx);
|
||||
|
||||
send_queued(&tx, queued, queued_bytes)
|
||||
.await
|
||||
.expect("first queue entry should fit");
|
||||
drain_queued_inflight_bytes(&mut rx).await;
|
||||
|
||||
let blocked = vec![Bytes::from_static(b"blocked")];
|
||||
let blocked_bytes = queued_block_bytes(&blocked);
|
||||
{
|
||||
let pending_send = send_queued(&tx, blocked, blocked_bytes);
|
||||
tokio::pin!(pending_send);
|
||||
assert!(
|
||||
futures::poll!(pending_send.as_mut()).is_pending(),
|
||||
"full queue must suspend producer send"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline + u64::try_from(queued_bytes).expect("queue bytes fit the gauge"),
|
||||
"dropping a pending producer send must compensate its bytes"
|
||||
);
|
||||
|
||||
drop(rx);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline,
|
||||
"dropping the receiver must settle every buffered entry"
|
||||
);
|
||||
|
||||
let rejected = vec![Bytes::from_static(b"rejected")];
|
||||
let rejected_bytes = queued_block_bytes(&rejected);
|
||||
assert!(
|
||||
send_queued(&tx, rejected, rejected_bytes).await.is_err(),
|
||||
"closed receiver must reject a new send"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline,
|
||||
"failed sends must compensate their bytes"
|
||||
);
|
||||
assert!(rx.recv().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn queued_batch_entry_settles_bytes_before_handoff() {
|
||||
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let (tx, rx) = mpsc::channel(2);
|
||||
let mut rx = rx;
|
||||
let batch = vec![vec![Bytes::from_static(b"queued")], vec![Bytes::from_static(b"batch")]];
|
||||
let batch_bytes = queued_batch_bytes(&batch);
|
||||
async fn drain_queued_batched_inflight_bytes_consumes_pending_batches() {
|
||||
let (tx, mut rx) = mpsc::channel(2);
|
||||
tx.send(vec![vec![Bytes::from_static(b"queued")]]).await.unwrap();
|
||||
drop(tx);
|
||||
|
||||
send_queued(&tx, batch, batch_bytes).await.expect("batch should be queued");
|
||||
let batch = rx.recv().await.expect("queued batch should be received").into_inner();
|
||||
assert_eq!(batch_bytes, queued_batch_bytes(&batch));
|
||||
drain_queued_batched_inflight_bytes(&mut rx).await;
|
||||
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline,
|
||||
"receiving a batch must settle all contained block bytes before shard writes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn queued_entry_settles_when_a_closed_receiver_accepts_an_outstanding_permit() {
|
||||
let baseline = rustfs_io_metrics::current_ec_encode_inflight_bytes();
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
let permit = tx
|
||||
.clone()
|
||||
.reserve_owned()
|
||||
.await
|
||||
.expect("open receiver should reserve queue capacity");
|
||||
rx.close();
|
||||
assert!(
|
||||
matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
|
||||
"an outstanding permit must leave the closed queue observably empty"
|
||||
);
|
||||
|
||||
let block = vec![Bytes::from_static(b"late-permit")];
|
||||
let block_bytes = queued_block_bytes(&block);
|
||||
permit.send(InflightEntry::new(block, block_bytes));
|
||||
drop(rx);
|
||||
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::current_ec_encode_inflight_bytes(),
|
||||
baseline,
|
||||
"a queue entry sent through an outstanding permit must settle when Tokio drops it"
|
||||
);
|
||||
assert!(rx.recv().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -640,7 +640,7 @@ impl Erasure {
|
||||
/// # Returns
|
||||
/// A vector of encoded shards as `Bytes`.
|
||||
#[tracing::instrument(level = "debug", skip_all, fields(data_len=data.len()))]
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub fn encode_data(&self, data: &[u8]) -> io::Result<Vec<Bytes>> {
|
||||
let shard_size_fn = if self.uses_legacy {
|
||||
calc_shard_size_legacy
|
||||
@@ -688,7 +688,7 @@ impl Erasure {
|
||||
|
||||
/// Encode owned data, avoiding a copy when the caller already has a heap buffer.
|
||||
/// Falls back to copying into a new buffer if zero-copy conversion fails.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub fn encode_data_owned(&self, data: Vec<u8>) -> io::Result<Vec<Bytes>> {
|
||||
let shard_size_fn = if self.uses_legacy {
|
||||
calc_shard_size_legacy
|
||||
@@ -752,7 +752,7 @@ impl Erasure {
|
||||
/// block), the `resize(need_total_size)` below stays within capacity for every
|
||||
/// `data_len <= block_size` — both shard-size formulas are monotone in
|
||||
/// `data_len` — so this function never reallocates the buffer.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub fn encode_data_bytes_mut(&self, mut data_buffer: BytesMut, data_len: usize) -> io::Result<Vec<Bytes>> {
|
||||
let shard_size_fn = if self.uses_legacy {
|
||||
calc_shard_size_legacy
|
||||
@@ -805,7 +805,7 @@ impl Erasure {
|
||||
///
|
||||
/// # Returns
|
||||
/// Ok if reconstruction succeeds, error otherwise.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub fn decode_data(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
|
||||
if self.parity_shards > 0 {
|
||||
if self.uses_legacy {
|
||||
@@ -825,7 +825,7 @@ impl Erasure {
|
||||
}
|
||||
|
||||
/// Decode and reconstruct missing data shards, then regenerate parity shards.
|
||||
#[hotpath::measure(impl_type = "Erasure")]
|
||||
#[hotpath::measure]
|
||||
pub fn decode_data_and_parity(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
|
||||
if self.parity_shards > 0 {
|
||||
if self.uses_legacy {
|
||||
|
||||
@@ -19,8 +19,6 @@ use crate::diagnostics::get::{
|
||||
GET_STAGE_READER_MMAP_PATH_RESOLVE, GET_STAGE_READER_OPEN_MMAP_COPY_FALLBACK, GET_STAGE_READER_OPEN_MMAP_COPY_SUCCESS,
|
||||
GET_STAGE_READER_OPEN_STREAM, GET_STAGE_READER_STREAM_FIRST_READ, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::disk::FileWriter;
|
||||
use crate::disk::{self, DiskAPI as _, DiskStore, FileReader, MmapCopyStageMetrics, error::DiskError};
|
||||
use crate::erasure::coding::{BitrotReader, BitrotWriterWrapper, CustomWriter};
|
||||
use bytes::Bytes;
|
||||
@@ -38,11 +36,6 @@ use std::time::Instant;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tracing::debug;
|
||||
|
||||
#[cfg(all(test, feature = "hotpath"))]
|
||||
tokio::task_local! {
|
||||
static FORCE_MMAP_COPY_FAILURE_FOR_TEST: ();
|
||||
}
|
||||
|
||||
/// A shard source for the bitrot reader.
|
||||
///
|
||||
/// `InMemory` keeps the `Bytes` concrete instead of erasing it behind
|
||||
@@ -367,21 +360,10 @@ async fn open_disk_reader(
|
||||
mmap_copy_stage: GET_STAGE_READER_MMAP_COPY_BUFFER,
|
||||
direct_read_copy_stage: GET_STAGE_READER_MMAP_DIRECT_READ_COPY,
|
||||
});
|
||||
let mmap_result = {
|
||||
#[cfg(all(test, feature = "hotpath"))]
|
||||
if FORCE_MMAP_COPY_FAILURE_FOR_TEST.try_with(|_| ()).is_ok() {
|
||||
Err(DiskError::other("forced mmap-copy failure for test"))
|
||||
} else {
|
||||
disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
|
||||
.await
|
||||
}
|
||||
#[cfg(not(all(test, feature = "hotpath")))]
|
||||
{
|
||||
disk.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
|
||||
.await
|
||||
}
|
||||
};
|
||||
match mmap_result {
|
||||
match disk
|
||||
.read_file_mmap_copy_with_metrics(bucket, path, offset, length, mmap_metrics)
|
||||
.await
|
||||
{
|
||||
Ok(bytes) => {
|
||||
let duration_ms = zero_copy_start.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
@@ -416,11 +398,7 @@ async fn open_disk_reader(
|
||||
}
|
||||
|
||||
return match stream_result {
|
||||
Ok(reader) => {
|
||||
#[cfg(feature = "hotpath")]
|
||||
let reader = instrument_raw_shard_reader(reader, disk.is_local());
|
||||
Ok(wrap_first_read_metrics(reader, metrics_path))
|
||||
}
|
||||
Ok(reader) => Ok(wrap_first_read_metrics(reader, metrics_path)),
|
||||
Err(_) => Err(err),
|
||||
};
|
||||
}
|
||||
@@ -429,8 +407,6 @@ async fn open_disk_reader(
|
||||
|
||||
let stream_start = stage_metrics_enabled.then(Instant::now);
|
||||
let reader = disk.read_file_stream(bucket, path, offset, length).await?;
|
||||
#[cfg(feature = "hotpath")]
|
||||
let reader = instrument_raw_shard_reader(reader, disk.is_local());
|
||||
if let Some(metrics_path) = metrics_path {
|
||||
record_get_stage_duration_if_enabled(metrics_path, GET_STAGE_READER_OPEN_STREAM, stream_start);
|
||||
}
|
||||
@@ -451,37 +427,6 @@ fn wrap_first_read_metrics(reader: FileReader, metrics_path: Option<&'static str
|
||||
ShardReader::Stream(reader)
|
||||
}
|
||||
|
||||
// The labels are deliberately fixed: object keys, disk paths, and remote hosts
|
||||
// are all high-cardinality or sensitive and belong nowhere in a profiling report.
|
||||
#[cfg(feature = "hotpath")]
|
||||
const RAW_SHARD_READ_LOCAL_LABEL: &str = "EC raw shard read local";
|
||||
#[cfg(feature = "hotpath")]
|
||||
const RAW_SHARD_READ_REMOTE_LABEL: &str = "EC raw shard read remote";
|
||||
#[cfg(feature = "hotpath")]
|
||||
const RAW_SHARD_WRITE_LOCAL_LABEL: &str = "EC raw shard write local";
|
||||
#[cfg(feature = "hotpath")]
|
||||
const RAW_SHARD_WRITE_REMOTE_LABEL: &str = "EC raw shard write remote";
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
fn instrument_raw_shard_reader(reader: FileReader, is_local: bool) -> FileReader {
|
||||
// `io!` aggregates by call site, so the local and remote branches must remain distinct.
|
||||
if is_local {
|
||||
Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_LOCAL_LABEL))
|
||||
} else {
|
||||
Box::new(hotpath::io!(reader, label = RAW_SHARD_READ_REMOTE_LABEL))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
fn instrument_raw_shard_writer(writer: FileWriter, is_local: bool) -> FileWriter {
|
||||
// `io!` aggregates by call site, so the local and remote branches must remain distinct.
|
||||
if is_local {
|
||||
Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_LOCAL_LABEL))
|
||||
} else {
|
||||
Box::new(hotpath::io!(writer, label = RAW_SHARD_WRITE_REMOTE_LABEL))
|
||||
}
|
||||
}
|
||||
|
||||
fn bitrot_encoded_range(offset: usize, length: usize, shard_size: usize, checksum_algo: HashAlgorithm) -> (usize, usize) {
|
||||
(
|
||||
offset.div_ceil(shard_size) * checksum_algo.size() + offset,
|
||||
@@ -741,8 +686,6 @@ pub async fn create_bitrot_writer(
|
||||
};
|
||||
|
||||
let file = disk.create_file("", volume, path, length).await?;
|
||||
#[cfg(feature = "hotpath")]
|
||||
let file = instrument_raw_shard_writer(file, disk.is_local());
|
||||
CustomWriter::new_tokio_writer(file)
|
||||
} else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
@@ -755,182 +698,6 @@ pub async fn create_bitrot_writer(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::cluster::rpc::RemoteDisk;
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
InternodeDataTransport, InternodeDataTransportCapabilities, ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
|
||||
};
|
||||
#[cfg(feature = "hotpath")]
|
||||
use crate::disk::{Disk, DiskOption, error::Result};
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
#[derive(Debug, Clone, Default)]
|
||||
struct TestRemoteDataTransport {
|
||||
bytes: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
impl TestRemoteDataTransport {
|
||||
fn bytes(&self) -> Vec<u8> {
|
||||
self.bytes
|
||||
.lock()
|
||||
.expect("test remote transport bytes lock should not be poisoned")
|
||||
.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
#[derive(Debug)]
|
||||
struct TestRemoteWriter {
|
||||
bytes: Arc<Mutex<Vec<u8>>>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
impl tokio::io::AsyncWrite for TestRemoteWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
self.bytes
|
||||
.lock()
|
||||
.expect("test remote transport bytes lock should not be poisoned")
|
||||
.extend_from_slice(buf);
|
||||
Poll::Ready(Ok(buf.len()))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
#[async_trait::async_trait]
|
||||
impl InternodeDataTransport for TestRemoteDataTransport {
|
||||
async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
|
||||
Ok(Box::new(Cursor::new(self.bytes())))
|
||||
}
|
||||
|
||||
async fn open_write(&self, _request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
Ok(Box::new(TestRemoteWriter {
|
||||
bytes: Arc::clone(&self.bytes),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn open_walk_dir(&self, _request: WalkDirStreamRequest) -> Result<FileReader> {
|
||||
panic!("open_walk_dir must not be used by the raw shard I/O test")
|
||||
}
|
||||
|
||||
fn name(&self) -> &'static str {
|
||||
"bitrot-test-remote"
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> InternodeDataTransportCapabilities {
|
||||
InternodeDataTransportCapabilities::tcp_http()
|
||||
}
|
||||
}
|
||||
|
||||
async fn local_test_disk() -> (DiskStore, tempfile::TempDir) {
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::{DiskOption, new_disk};
|
||||
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("local disk should be created");
|
||||
|
||||
(disk, dir)
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
async fn remote_test_disk() -> (DiskStore, TestRemoteDataTransport) {
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
|
||||
let endpoint = Endpoint {
|
||||
url: url::Url::parse("http://remote-node:9000/data/rustfs0").expect("test remote endpoint should parse"),
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: 0,
|
||||
};
|
||||
let transport = TestRemoteDataTransport::default();
|
||||
let remote = RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(transport.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("test remote disk should be created");
|
||||
|
||||
(Arc::new(Disk::Remote(Box::new(remote))), transport)
|
||||
}
|
||||
|
||||
async fn round_trip_disk_bitrot(disk: &DiskStore, bucket: &str, path: &str, payload: &[u8], shard_size: usize) -> Vec<u8> {
|
||||
disk.make_volume(bucket).await.expect("volume should be created");
|
||||
let mut writer = create_bitrot_writer(
|
||||
false,
|
||||
Some(disk),
|
||||
bucket,
|
||||
path,
|
||||
i64::try_from(payload.len()).expect("test payload length should fit i64"),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
)
|
||||
.await
|
||||
.expect("disk bitrot writer should open the raw shard file");
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer
|
||||
.write(chunk)
|
||||
.await
|
||||
.expect("disk bitrot writer should preserve each shard block");
|
||||
}
|
||||
writer.shutdown().await.expect("disk bitrot writer should close cleanly");
|
||||
|
||||
let mut reader = create_bitrot_reader(
|
||||
None,
|
||||
Some(disk),
|
||||
bucket,
|
||||
path,
|
||||
0,
|
||||
payload.len(),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.expect("disk bitrot reader should open the raw shard file")
|
||||
.expect("disk bitrot reader should exist");
|
||||
let mut actual = Vec::with_capacity(payload.len());
|
||||
while actual.len() < payload.len() {
|
||||
let remaining = payload.len() - actual.len();
|
||||
let mut chunk = vec![0; remaining.min(shard_size)];
|
||||
let read = reader
|
||||
.read(&mut chunk)
|
||||
.await
|
||||
.expect("disk bitrot reader should return the complete shard body");
|
||||
assert!(read > 0, "disk bitrot reader must not end before the expected shard body is complete");
|
||||
actual.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
|
||||
actual
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_mmap_read_enabled_accepts_legacy_zero_copy_alias() {
|
||||
temp_env::with_vars(
|
||||
@@ -957,165 +724,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
#[test]
|
||||
fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes() {
|
||||
const CHILD_ENV: &str = "RUSTFS_HOTPATH_RAW_SHARD_IO_TEST_CHILD";
|
||||
if std::env::var_os(CHILD_ENV).is_none() {
|
||||
let status = std::process::Command::new(std::env::current_exe().expect("test executable path should be available"))
|
||||
.arg("--exact")
|
||||
.arg("io_support::bitrot::tests::raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes")
|
||||
.arg("--nocapture")
|
||||
.env(CHILD_ENV, "1")
|
||||
.status()
|
||||
.expect("isolated HotPath I/O test process should start");
|
||||
assert!(status.success(), "isolated HotPath I/O test process should pass");
|
||||
return;
|
||||
}
|
||||
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime should be created")
|
||||
.block_on(raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process());
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
async fn raw_shard_io_wrappers_report_fixed_labels_and_preserve_bytes_in_isolated_process() {
|
||||
use hotpath::{Format, HotpathGuardBuilder, Section};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let report_dir = tempfile::tempdir().expect("report tempdir should be created");
|
||||
let report_path = report_dir.path().join("hotpath-io.json");
|
||||
let guard = HotpathGuardBuilder::new("raw_shard_io_test")
|
||||
.format(Format::Json)
|
||||
.output_path(&report_path)
|
||||
.sections(vec![Section::Io])
|
||||
.build();
|
||||
|
||||
let (disk, _dir) = local_test_disk().await;
|
||||
let bucket = "test-bucket";
|
||||
let path = "obj/hotpath-part.1";
|
||||
let payload = b"local shard bytes";
|
||||
let shard_size = 4;
|
||||
let local_read = round_trip_disk_bitrot(&disk, bucket, path, payload, shard_size).await;
|
||||
assert_eq!(local_read, payload, "local raw shard I/O instrumentation must not alter stored bytes");
|
||||
|
||||
let fallback_path = "obj/hotpath-mmap-fallback-part.1";
|
||||
let fallback_payload = b"mmap fallback shard bytes";
|
||||
disk.write_all("test-bucket", fallback_path, Bytes::from_static(fallback_payload))
|
||||
.await
|
||||
.expect("fallback shard file should be written");
|
||||
let mut fallback_reader = FORCE_MMAP_COPY_FAILURE_FOR_TEST
|
||||
.scope((), open_disk_reader(&disk, bucket, fallback_path, 0, fallback_payload.len(), true, None))
|
||||
.await
|
||||
.expect("mmap-copy failure should fall back to a raw shard stream");
|
||||
assert!(
|
||||
matches!(fallback_reader, ShardReader::Stream(_)),
|
||||
"forced mmap-copy failure must use the streaming fallback"
|
||||
);
|
||||
let mut fallback_read = Vec::new();
|
||||
fallback_reader
|
||||
.read_to_end(&mut fallback_read)
|
||||
.await
|
||||
.expect("mmap-copy fallback stream should preserve bytes");
|
||||
assert_eq!(fallback_read, fallback_payload);
|
||||
|
||||
let (remote_disk, remote_transport) = remote_test_disk().await;
|
||||
let remote_payload = b"remote shard bytes";
|
||||
let mut remote_writer = create_bitrot_writer(
|
||||
false,
|
||||
Some(&remote_disk),
|
||||
bucket,
|
||||
"obj/hotpath-remote-part.1",
|
||||
i64::try_from(remote_payload.len()).expect("remote payload length should fit i64"),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
)
|
||||
.await
|
||||
.expect("remote bitrot writer should use the production raw writer path");
|
||||
for chunk in remote_payload.chunks(shard_size) {
|
||||
remote_writer
|
||||
.write(chunk)
|
||||
.await
|
||||
.expect("remote bitrot writer should preserve bytes");
|
||||
}
|
||||
remote_writer
|
||||
.shutdown()
|
||||
.await
|
||||
.expect("remote bitrot writer should close cleanly");
|
||||
assert_eq!(remote_transport.bytes(), remote_payload);
|
||||
|
||||
let mut reader = create_bitrot_reader(
|
||||
None,
|
||||
Some(&remote_disk),
|
||||
bucket,
|
||||
"obj/hotpath-remote-part.1",
|
||||
0,
|
||||
remote_payload.len(),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("remote bitrot reader should use the production raw reader path")
|
||||
.expect("remote bitrot reader should exist");
|
||||
let mut remote_read = Vec::with_capacity(remote_payload.len());
|
||||
while remote_read.len() < remote_payload.len() {
|
||||
let remaining = remote_payload.len() - remote_read.len();
|
||||
let mut chunk = vec![0; remaining.min(shard_size)];
|
||||
let read = reader
|
||||
.read(&mut chunk)
|
||||
.await
|
||||
.expect("remote bitrot reader should preserve bytes");
|
||||
assert!(read > 0, "remote bitrot reader must not end before the expected shard body is complete");
|
||||
remote_read.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
assert_eq!(remote_read, remote_payload);
|
||||
|
||||
drop(guard);
|
||||
let report = std::fs::read_to_string(&report_path).expect("HotPath I/O report should be written");
|
||||
let report: serde_json::Value = serde_json::from_str(&report).expect("HotPath I/O report should be valid JSON");
|
||||
let entries = report["io"]["data"]
|
||||
.as_array()
|
||||
.expect("HotPath I/O report should include data rows");
|
||||
let io_bytes = |label: &str, direction: &str| {
|
||||
let byte_count: u64 = entries
|
||||
.iter()
|
||||
.filter(|entry| entry["label"].as_str().is_some_and(|entry_label| entry_label == label))
|
||||
.filter_map(|entry| entry[direction]["bytes"].as_u64())
|
||||
.sum();
|
||||
assert!(byte_count > 0, "report must include fixed label {label}");
|
||||
byte_count
|
||||
};
|
||||
let payload_len = u64::try_from(payload.len()).expect("test payload length should fit u64");
|
||||
assert_eq!(
|
||||
io_bytes(RAW_SHARD_READ_LOCAL_LABEL, "read"),
|
||||
payload_len + u64::try_from(fallback_payload.len()).expect("fallback payload length should fit u64")
|
||||
);
|
||||
assert_eq!(io_bytes(RAW_SHARD_WRITE_LOCAL_LABEL, "write"), payload_len);
|
||||
assert_eq!(
|
||||
io_bytes(RAW_SHARD_READ_REMOTE_LABEL, "read"),
|
||||
u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
|
||||
);
|
||||
assert_eq!(
|
||||
io_bytes(RAW_SHARD_WRITE_REMOTE_LABEL, "write"),
|
||||
u64::try_from(remote_payload.len()).expect("remote payload length should fit u64")
|
||||
);
|
||||
for label in [
|
||||
RAW_SHARD_READ_LOCAL_LABEL,
|
||||
RAW_SHARD_READ_REMOTE_LABEL,
|
||||
RAW_SHARD_WRITE_LOCAL_LABEL,
|
||||
RAW_SHARD_WRITE_REMOTE_LABEL,
|
||||
] {
|
||||
assert!(
|
||||
!label.contains(['/', ':', '?', '@']),
|
||||
"raw shard I/O labels must not carry a path, host, query, or credential delimiter: {label}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_mmap_read_max_length_defaults_and_env_override() {
|
||||
temp_env::with_var(ENV_OBJECT_MMAP_READ_MAX_LENGTH, None::<&str>, || {
|
||||
@@ -1133,9 +741,25 @@ mod tests {
|
||||
// be materialized in memory by the mmap-copy path; over-cap reads stream.
|
||||
#[tokio::test]
|
||||
async fn open_disk_reader_streams_when_length_exceeds_mmap_cap() {
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::disk::{DiskOption, new_disk};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (disk, _dir) = local_test_disk().await;
|
||||
let dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(dir.path().to_str().expect("tempdir path should be utf8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("local disk should be created");
|
||||
|
||||
let payload = vec![7u8; 4096];
|
||||
disk.make_volume("test-bucket").await.expect("volume should be created");
|
||||
@@ -1190,17 +814,6 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disk_bitrot_reader_and_writer_preserve_full_shard_body() {
|
||||
let (disk, _dir) = local_test_disk().await;
|
||||
let bucket = "test-bucket";
|
||||
let path = "obj/wrapped-part.1";
|
||||
let payload = b"wrapped shard body";
|
||||
let actual = round_trip_disk_bitrot(&disk, bucket, path, payload, 4).await;
|
||||
|
||||
assert_eq!(actual, payload, "raw shard I/O instrumentation must not alter stored bytes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_bitrot_reader_with_inline_data() {
|
||||
let test_data = b"hello world test data";
|
||||
|
||||
@@ -337,15 +337,6 @@ pub struct NotificationPeerErr {
|
||||
pub err: Option<Error>,
|
||||
}
|
||||
|
||||
/// One peer's answer to a KMS configuration fingerprint probe.
|
||||
pub struct PeerKmsConfigFingerprint {
|
||||
pub host: String,
|
||||
/// `None` when the peer has no KMS configuration of its own, or could not
|
||||
/// be asked at all, in which case `err` carries the reason.
|
||||
pub fingerprint: Option<String>,
|
||||
pub err: Option<Error>,
|
||||
}
|
||||
|
||||
fn notification_peer_result<T>(host: String, result: Result<T>) -> NotificationPeerErr {
|
||||
NotificationPeerErr { host, err: result.err() }
|
||||
}
|
||||
@@ -527,47 +518,6 @@ impl NotificationSys {
|
||||
self.signal_dynamic_config(sub_sys, false).await
|
||||
}
|
||||
|
||||
/// Ask every peer to re-read the cluster-persisted KMS configuration.
|
||||
///
|
||||
/// Best-effort by contract: the caller has already switched locally, so a
|
||||
/// peer that fails is reported rather than rolled back. Peers built before
|
||||
/// the KMS subsystem existed reject the signal with an explicit error.
|
||||
pub async fn reload_kms_config(&self) -> Vec<NotificationPeerErr> {
|
||||
self.reload_dynamic_config(crate::cluster::rpc::KMS_SIGNAL_SUBSYSTEM).await
|
||||
}
|
||||
|
||||
/// Collect the KMS configuration fingerprint each peer is running.
|
||||
///
|
||||
/// A peer whose build predates the KMS subsystem rejects the probe, so it
|
||||
/// is reported as an error rather than silently agreeing with this node.
|
||||
pub async fn kms_config_fingerprints(&self) -> Vec<PeerKmsConfigFingerprint> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter() {
|
||||
futures.push(async move {
|
||||
let Some(client) = client else {
|
||||
return PeerKmsConfigFingerprint {
|
||||
host: String::new(),
|
||||
fingerprint: None,
|
||||
err: Some(Error::other("peer is not reachable")),
|
||||
};
|
||||
};
|
||||
match client.kms_config_fingerprint().await {
|
||||
Ok(fingerprint) => PeerKmsConfigFingerprint {
|
||||
host: client.host.to_string(),
|
||||
fingerprint,
|
||||
err: None,
|
||||
},
|
||||
Err(e) => PeerKmsConfigFingerprint {
|
||||
host: client.host.to_string(),
|
||||
fingerprint: None,
|
||||
err: Some(e),
|
||||
},
|
||||
}
|
||||
});
|
||||
}
|
||||
join_all(futures).await
|
||||
}
|
||||
|
||||
pub async fn refresh_config_snapshot(&self) -> Vec<NotificationPeerErr> {
|
||||
let mut futures = Vec::with_capacity(self.peer_clients.len());
|
||||
for client in self.peer_clients.iter() {
|
||||
|
||||
@@ -199,7 +199,7 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
#[hotpath::measure]
|
||||
pub async fn read_version_optimized(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -238,7 +238,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
#[hotpath::measure]
|
||||
pub(super) async fn get_object_fileinfo(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -410,7 +410,7 @@ impl SetDisks {
|
||||
Ok((fi, parts_metadata, op_online_disks))
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
#[hotpath::measure]
|
||||
pub(super) async fn get_object_info_and_quorum(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -605,7 +605,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
#[hotpath::measure]
|
||||
pub(super) async fn get_object_with_fileinfo<W>(
|
||||
// &self,
|
||||
bucket: &str,
|
||||
@@ -1140,7 +1140,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
#[hotpath::measure]
|
||||
pub(super) async fn get_object_decode_reader_with_fileinfo(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -1296,7 +1296,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
#[hotpath::measure]
|
||||
async fn build_codec_streaming_part_reader(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
|
||||
@@ -238,7 +238,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
#[instrument(skip(self, data))]
|
||||
#[hotpath::measure(impl_type = "ECStore")]
|
||||
#[hotpath::measure]
|
||||
pub(super) async fn handle_put_object_part(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -815,7 +815,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
#[hotpath::measure(impl_type = "ECStore")]
|
||||
#[hotpath::measure]
|
||||
pub(super) async fn handle_get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -849,7 +849,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, data))]
|
||||
#[hotpath::measure(impl_type = "ECStore")]
|
||||
#[hotpath::measure]
|
||||
pub(super) async fn handle_put_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
const STORE_MULTIPART: &str = include_str!("../src/store/multipart.rs");
|
||||
const STORE_OBJECT: &str = include_str!("../src/store/object.rs");
|
||||
const ERASURE: &str = include_str!("../src/erasure/coding/erasure.rs");
|
||||
const DECODE: &str = include_str!("../src/erasure/coding/decode.rs");
|
||||
const ENCODE: &str = include_str!("../src/erasure/coding/encode.rs");
|
||||
const LOCAL_DISK: &str = include_str!("../src/disk/local.rs");
|
||||
const BITROT: &str = include_str!("../src/erasure/coding/bitrot.rs");
|
||||
const SET_DISK_READ: &str = include_str!("../src/set_disk/read.rs");
|
||||
|
||||
fn assert_measured_as(source: &str, impl_type: &str, function: &str) {
|
||||
let attribute = format!("#[hotpath::measure(impl_type = \"{impl_type}\")]\n");
|
||||
let function_start = format!("fn {function}");
|
||||
let mut remaining = source;
|
||||
|
||||
while let Some(attribute_offset) = remaining.find(&attribute) {
|
||||
let measured_source = &remaining[attribute_offset + attribute.len()..];
|
||||
if measured_source.find(&function_start).is_some_and(|offset| offset < 256) {
|
||||
return;
|
||||
}
|
||||
remaining = measured_source;
|
||||
}
|
||||
|
||||
panic!("missing HotPath impl_type={impl_type} for {function}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inherent_hotpath_measurements_have_cpu_attribution_types() {
|
||||
assert_measured_as(STORE_MULTIPART, "ECStore", "handle_put_object_part");
|
||||
for function in ["handle_get_object_reader", "handle_put_object"] {
|
||||
assert_measured_as(STORE_OBJECT, "ECStore", function);
|
||||
}
|
||||
for function in [
|
||||
"encode_data",
|
||||
"encode_data_owned",
|
||||
"encode_data_bytes_mut",
|
||||
"decode_data",
|
||||
"decode_data_and_parity",
|
||||
] {
|
||||
assert_measured_as(ERASURE, "Erasure", function);
|
||||
}
|
||||
assert_measured_as(DECODE, "ParallelReader", "read");
|
||||
assert_measured_as(DECODE, "Erasure", "decode");
|
||||
for function in [
|
||||
"encode",
|
||||
"encode_batched",
|
||||
"encode_inline_small",
|
||||
"encode_single_block_non_inline",
|
||||
] {
|
||||
assert_measured_as(ENCODE, "Erasure", function);
|
||||
}
|
||||
for function in ["read_metadata_with_dmtime", "read_all_data"] {
|
||||
assert_measured_as(LOCAL_DISK, "LocalDisk", function);
|
||||
}
|
||||
assert_measured_as(BITROT, "BitrotReader", "read");
|
||||
assert!(
|
||||
BITROT.contains("#[hotpath::measure(label = \"BitrotWriter::write\", impl_type = \"BitrotWriter\")]"),
|
||||
"BitrotWriter::write must retain its stable label and CPU impl_type",
|
||||
);
|
||||
for function in [
|
||||
"read_version_optimized",
|
||||
"get_object_fileinfo",
|
||||
"get_object_info_and_quorum",
|
||||
"get_object_with_fileinfo",
|
||||
"get_object_decode_reader_with_fileinfo",
|
||||
"build_codec_streaming_part_reader",
|
||||
] {
|
||||
assert_measured_as(SET_DISK_READ, "SetDisks", function);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn module_hotpath_measurements_remain_untyped() {
|
||||
assert!(
|
||||
BITROT.contains("#[hotpath::measure]\npub async fn bitrot_verify"),
|
||||
"bitrot_verify is a module function and must not claim an impl type",
|
||||
);
|
||||
assert!(
|
||||
SET_DISK_READ.contains("#[hotpath::measure]\nasync fn setup_multipart_part_readers"),
|
||||
"setup_multipart_part_readers is a module function and must not claim an impl type",
|
||||
);
|
||||
}
|
||||
+30
-239
@@ -41,7 +41,7 @@ use rustfs_policy::policy::Args;
|
||||
use rustfs_policy::policy::opa;
|
||||
use rustfs_policy::policy::{Policy, PolicyDoc, iam_policy_claim_name_sa, policy_needs_existing_object_tag_for_args};
|
||||
use serde_json::Value;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use time::OffsetDateTime;
|
||||
@@ -61,51 +61,10 @@ const VERIFIED_FEDERATED_POLICY_CLAIM: &str = "x-rustfs-internal-federated-polic
|
||||
const FEDERATED_POLICY_BOUNDARY_CLAIM: &str = "x-rustfs-internal-federated-boundary";
|
||||
pub(crate) const SITE_REPLICATOR_CLAIM: &str = "site-replicator";
|
||||
|
||||
#[derive(Clone)]
|
||||
enum PolicyPluginState {
|
||||
Disabled,
|
||||
Initializing,
|
||||
Ready(opa::AuthZPlugin),
|
||||
Failed,
|
||||
}
|
||||
static POLICY_PLUGIN_CLIENT: OnceLock<Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>>> = OnceLock::new();
|
||||
|
||||
static POLICY_PLUGIN_STATE: OnceLock<Arc<RwLock<PolicyPluginState>>> = OnceLock::new();
|
||||
|
||||
fn get_policy_plugin_state() -> Arc<RwLock<PolicyPluginState>> {
|
||||
POLICY_PLUGIN_STATE
|
||||
.get_or_init(|| {
|
||||
let configured = opa::is_configured();
|
||||
let state = Arc::new(RwLock::new(if configured {
|
||||
PolicyPluginState::Initializing
|
||||
} else {
|
||||
PolicyPluginState::Disabled
|
||||
}));
|
||||
if configured {
|
||||
let state = Arc::clone(&state);
|
||||
tokio::spawn(async move {
|
||||
let next_state = match opa::lookup_config().await {
|
||||
Ok(conf) if conf.enable() => {
|
||||
info!("OPA plugin enabled");
|
||||
PolicyPluginState::Ready(opa::AuthZPlugin::new(conf))
|
||||
}
|
||||
Ok(_) => PolicyPluginState::Failed,
|
||||
Err(e) => {
|
||||
error!(
|
||||
component = "iam",
|
||||
subsystem = "policy_plugin",
|
||||
result = "configuration_load_failed",
|
||||
error_kind = e.kind(),
|
||||
"OPA plugin configuration load failed"
|
||||
);
|
||||
PolicyPluginState::Failed
|
||||
}
|
||||
};
|
||||
*state.write().await = next_state;
|
||||
});
|
||||
}
|
||||
state
|
||||
})
|
||||
.clone()
|
||||
fn get_policy_plugin_client() -> Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>> {
|
||||
POLICY_PLUGIN_CLIENT.get_or_init(|| Arc::new(RwLock::new(None))).clone()
|
||||
}
|
||||
|
||||
pub struct IamSys<T> {
|
||||
@@ -214,7 +173,19 @@ impl<T: Store> IamSys<T> {
|
||||
/// # Returns
|
||||
/// A new instance of IamSys
|
||||
pub fn new(store: Arc<IamCache<T>>) -> Self {
|
||||
get_policy_plugin_state();
|
||||
tokio::spawn(async move {
|
||||
match opa::lookup_config().await {
|
||||
Ok(conf) => {
|
||||
if conf.enable() {
|
||||
Self::set_policy_plugin_client(opa::AuthZPlugin::new(conf)).await;
|
||||
info!("OPA plugin enabled");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Error loading OPA configuration err:{}", e);
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
Self {
|
||||
store,
|
||||
@@ -235,22 +206,15 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
|
||||
pub async fn set_policy_plugin_client(client: rustfs_policy::policy::opa::AuthZPlugin) {
|
||||
let policy_plugin_state = get_policy_plugin_state();
|
||||
let mut guard = policy_plugin_state.write().await;
|
||||
*guard = PolicyPluginState::Ready(client);
|
||||
let policy_plugin_client = get_policy_plugin_client();
|
||||
let mut guard = policy_plugin_client.write().await;
|
||||
*guard = Some(client);
|
||||
}
|
||||
|
||||
pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
|
||||
let policy_plugin_state = get_policy_plugin_state();
|
||||
let guard = policy_plugin_state.read().await;
|
||||
match &*guard {
|
||||
PolicyPluginState::Ready(client) => Some(client.clone()),
|
||||
PolicyPluginState::Disabled | PolicyPluginState::Initializing | PolicyPluginState::Failed => None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn policy_plugin_state() -> PolicyPluginState {
|
||||
get_policy_plugin_state().read().await.clone()
|
||||
let policy_plugin_client = get_policy_plugin_client();
|
||||
let guard = policy_plugin_client.read().await;
|
||||
guard.clone()
|
||||
}
|
||||
|
||||
pub async fn load_group(&self, name: &str) -> Result<()> {
|
||||
@@ -1096,20 +1060,11 @@ impl<T: Store> IamSys<T> {
|
||||
};
|
||||
}
|
||||
|
||||
match Self::policy_plugin_state().await {
|
||||
PolicyPluginState::Ready(_) => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Opa,
|
||||
};
|
||||
}
|
||||
PolicyPluginState::Initializing | PolicyPluginState::Failed => {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
PolicyPluginState::Disabled => {}
|
||||
if Self::get_policy_plugin_client().await.is_some() {
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Opa,
|
||||
};
|
||||
}
|
||||
|
||||
let Ok((is_svc, parent_user)) = self.is_service_account(args.account).await else {
|
||||
@@ -1208,22 +1163,7 @@ impl<T: Store> IamSys<T> {
|
||||
};
|
||||
}
|
||||
|
||||
let (resolved_policies, combined_policy) = self.store.merge_policies(&policies.join(",")).await;
|
||||
if args.deny_only {
|
||||
let resolved_policy_names: HashSet<&str> = resolved_policies
|
||||
.split(',')
|
||||
.filter(|policy_name| !policy_name.trim().is_empty())
|
||||
.collect();
|
||||
if policies
|
||||
.iter()
|
||||
.any(|policy_name| !resolved_policy_names.contains(policy_name.as_str()))
|
||||
{
|
||||
return PreparedIamAuth {
|
||||
needs_existing_object_tag: false,
|
||||
mode: PreparedIamMode::Deny,
|
||||
};
|
||||
}
|
||||
}
|
||||
let combined_policy = self.get_combined_policy(&policies).await;
|
||||
PreparedIamAuth {
|
||||
needs_existing_object_tag: policy_needs_existing_object_tag_for_args(&combined_policy, args).await,
|
||||
mode: PreparedIamMode::Regular { combined_policy },
|
||||
@@ -1763,7 +1703,7 @@ mod tests {
|
||||
fn deprecated_list_polices_api_is_available() {
|
||||
let _ = IamSys::<StsTestMockStore>::list_polices;
|
||||
}
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action, StsAction};
|
||||
use rustfs_policy::policy::action::{Action, AdminAction, S3Action};
|
||||
use rustfs_policy::policy::policy_uses_existing_object_tag_conditions;
|
||||
use serde_json::Value;
|
||||
use std::{
|
||||
@@ -3369,155 +3309,6 @@ mod tests {
|
||||
assert!(!iam_sys.eval_prepared(&prepared, &args).await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_deny_only_rejects_unresolved_mapped_policy() {
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let user = "regular-unresolved-policy";
|
||||
let identity = UserIdentity::from(Credentials {
|
||||
access_key: user.to_string(),
|
||||
secret_key: "longenoughsecret".to_string(),
|
||||
status: ACCOUNT_ON.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
iam_sys.store.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_user(user, &identity, now);
|
||||
cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
|
||||
});
|
||||
|
||||
let groups = None;
|
||||
let claims = HashMap::new();
|
||||
let conditions = HashMap::new();
|
||||
let args = Args {
|
||||
account: user,
|
||||
groups: &groups,
|
||||
action: Action::StsAction(StsAction::AssumeRoleAction),
|
||||
bucket: "",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
object: "",
|
||||
claims: &claims,
|
||||
deny_only: true,
|
||||
};
|
||||
|
||||
let prepared = iam_sys.prepare_regular_auth(&args).await;
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::Deny));
|
||||
assert!(
|
||||
!iam_sys.eval_prepared(&prepared, &args).await,
|
||||
"deny-only evaluation must fail closed when a mapped policy document is unresolved"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_full_evaluation_keeps_resolved_part_of_partial_mapping() {
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let user = "regular-partial-policy";
|
||||
let identity = UserIdentity::from(Credentials {
|
||||
access_key: user.to_string(),
|
||||
secret_key: "longenoughsecret".to_string(),
|
||||
status: ACCOUNT_ON.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
iam_sys.store.cache.with_write_lock(|cache| {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
cache.add_or_update_user(user, &identity, now);
|
||||
cache.add_or_update_user_policy(user, &MappedPolicy::new("readwrite,missing-policy"), now);
|
||||
});
|
||||
|
||||
let groups = None;
|
||||
let claims = HashMap::new();
|
||||
let conditions = HashMap::new();
|
||||
let args = Args {
|
||||
account: user,
|
||||
groups: &groups,
|
||||
action: Action::S3Action(S3Action::GetObjectAction),
|
||||
bucket: "bucket",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
object: "object",
|
||||
claims: &claims,
|
||||
deny_only: false,
|
||||
};
|
||||
|
||||
let prepared = iam_sys.prepare_regular_auth(&args).await;
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
|
||||
assert!(
|
||||
iam_sys.eval_prepared(&prepared, &args).await,
|
||||
"full evaluation must preserve the historical partial-resolution behavior"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_deny_only_honors_group_derived_explicit_deny() {
|
||||
let iam_sys = test_iam_sys().await;
|
||||
let deny_policy =
|
||||
Policy::parse_config(br#"{"Version":"2012-10-17","Statement":[{"Effect":"Deny","Action":["sts:AssumeRole"]}]}"#)
|
||||
.expect("group deny policy should parse");
|
||||
let now = OffsetDateTime::now_utc();
|
||||
iam_sys
|
||||
.store
|
||||
.cache
|
||||
.add_or_update_policy_doc("deny-assume-role", &PolicyDoc::new(deny_policy), now);
|
||||
iam_sys
|
||||
.store
|
||||
.cache
|
||||
.add_or_update_group_policy("testgroup", &MappedPolicy::new("deny-assume-role"), now);
|
||||
|
||||
let groups = Some(vec!["testgroup".to_string()]);
|
||||
let claims = HashMap::new();
|
||||
let conditions = HashMap::new();
|
||||
let args = Args {
|
||||
account: "sts-fallback-test-parent",
|
||||
groups: &groups,
|
||||
action: Action::StsAction(StsAction::AssumeRoleAction),
|
||||
bucket: "",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
object: "",
|
||||
claims: &claims,
|
||||
deny_only: true,
|
||||
};
|
||||
|
||||
let prepared = iam_sys.prepare_regular_auth(&args).await;
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::Regular { .. }));
|
||||
assert!(
|
||||
!iam_sys.eval_prepared(&prepared, &args).await,
|
||||
"group-derived explicit Deny must remain authoritative"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_deny_only_rejects_unresolved_group_mapping() {
|
||||
let iam_sys = test_iam_sys().await;
|
||||
iam_sys.store.cache.add_or_update_group_policy(
|
||||
"testgroup",
|
||||
&MappedPolicy::new("readwrite,missing-policy"),
|
||||
OffsetDateTime::now_utc(),
|
||||
);
|
||||
|
||||
let groups = Some(vec!["testgroup".to_string()]);
|
||||
let claims = HashMap::new();
|
||||
let conditions = HashMap::new();
|
||||
let args = Args {
|
||||
account: "sts-fallback-test-parent",
|
||||
groups: &groups,
|
||||
action: Action::StsAction(StsAction::AssumeRoleAction),
|
||||
bucket: "",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
object: "",
|
||||
claims: &claims,
|
||||
deny_only: true,
|
||||
};
|
||||
|
||||
let prepared = iam_sys.prepare_regular_auth(&args).await;
|
||||
assert!(matches!(prepared.mode, PreparedIamMode::Deny));
|
||||
assert!(
|
||||
!iam_sys.eval_prepared(&prepared, &args).await,
|
||||
"deny-only evaluation must fail closed for an unresolved group-derived mapping"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_sts_claim_policy_ignores_unsafe_and_missing_policy_names() {
|
||||
let store = StsTestMockStore::new(true);
|
||||
|
||||
@@ -2195,22 +2195,18 @@ pub fn record_allocator_memory_observation(backend: &'static str, observation: A
|
||||
}
|
||||
}
|
||||
|
||||
/// Track encoded bytes in the queue hand-off between erasure encode and disk writers.
|
||||
///
|
||||
/// This is queue occupancy, not a per-request or process-RSS memory limit. The
|
||||
/// erasure encoder settles it on failed/cancelled sends, receiver drop, and
|
||||
/// consumer hand-off before shard writes begin.
|
||||
/// Track encoded bytes currently queued between erasure encode and disk writers.
|
||||
#[inline(always)]
|
||||
pub fn add_ec_encode_inflight_bytes(bytes: usize) {
|
||||
let next = EC_ENCODE_INFLIGHT_BYTES.fetch_add(bytes as u64, Ordering::Relaxed) + bytes as u64;
|
||||
gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
|
||||
gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
|
||||
}
|
||||
|
||||
/// Remove encoded bytes from the tracked erasure encode in-flight gauge.
|
||||
#[inline(always)]
|
||||
pub fn remove_ec_encode_inflight_bytes(bytes: usize) {
|
||||
let next = saturating_sub_atomic(&EC_ENCODE_INFLIGHT_BYTES, bytes as u64);
|
||||
gauge_set_cached!("rustfs_ec_encode_inflight_bytes_current", next as f64);
|
||||
gauge!("rustfs_ec_encode_inflight_bytes_current").set(next as f64);
|
||||
}
|
||||
|
||||
/// Return the current tracked EC encode in-flight bytes.
|
||||
|
||||
@@ -64,10 +64,6 @@ md-5 = { workspace = true }
|
||||
arc-swap = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
rustfs-security-governance = { workspace = true }
|
||||
# `EventName` for KMS audit records. A leaf crate with no rustfs dependencies,
|
||||
# so the audit sink can live outside this crate without a second, drifting
|
||||
# copy of the event vocabulary.
|
||||
rustfs-s3-types = { workspace = true }
|
||||
|
||||
# HTTP client for Vault
|
||||
reqwest = { workspace = true }
|
||||
@@ -77,16 +73,6 @@ vaultrs = { workspace = true }
|
||||
rustify = { workspace = true }
|
||||
tokio-util = { workspace = true }
|
||||
|
||||
# AWS KMS backend. Credentials come from the standard aws-config provider chain
|
||||
# (environment, shared profile, IMDS/container roles); this crate never handles
|
||||
# AWS credential material itself.
|
||||
aws-config = { workspace = true }
|
||||
aws-sdk-kms = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
# SdkError variants and raw HTTP status are needed to classify AWS failures for
|
||||
# the operation policy's retry decisions.
|
||||
aws-smithy-runtime-api = { workspace = true, features = ["http-1x"] }
|
||||
aws-smithy-types = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
anyhow = { workspace = true }
|
||||
# Debugging recorder for asserting emitted metrics in tests.
|
||||
@@ -96,9 +82,6 @@ tempfile = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
# "net" backs the scripted loopback Vault used by the policy wiring tests.
|
||||
tokio = { workspace = true, features = ["net", "test-util"] }
|
||||
# Replays canned AWS KMS HTTP exchanges so the AWS backend tests stay offline.
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["test-util"] }
|
||||
http = { workspace = true }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
|
||||
@@ -227,12 +227,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
|
||||
// Step 12: Check cache statistics
|
||||
println!("12. Checking cache statistics...");
|
||||
if let Some(stats) = encryption_service.cache_stats().await {
|
||||
if let Some((hits, misses)) = encryption_service.cache_stats().await {
|
||||
println!(" ✓ Cache statistics:");
|
||||
println!(" - Cached entries: {}", stats.entries);
|
||||
println!(" - Cache hits: {}", stats.hits);
|
||||
println!(" - Cache misses: {}", stats.misses);
|
||||
println!(" - Entries evicted: {}\n", stats.evictions);
|
||||
println!(" - Cache hits: {}", hits);
|
||||
println!(" - Cache misses: {}\n", misses);
|
||||
} else {
|
||||
println!(" - Cache is disabled\n");
|
||||
}
|
||||
|
||||
@@ -263,12 +263,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
|
||||
// Step 12: Check cache statistics
|
||||
println!("12. Checking cache statistics...");
|
||||
if let Some(stats) = encryption_service.cache_stats().await {
|
||||
if let Some((hits, misses)) = encryption_service.cache_stats().await {
|
||||
println!(" ✓ Cache statistics:");
|
||||
println!(" - Cached entries: {}", stats.entries);
|
||||
println!(" - Cache hits: {}", stats.hits);
|
||||
println!(" - Cache misses: {}", stats.misses);
|
||||
println!(" - Entries evicted: {}\n", stats.evictions);
|
||||
println!(" - Cache hits: {}", hits);
|
||||
println!(" - Cache misses: {}\n", misses);
|
||||
} else {
|
||||
println!(" - Cache is disabled\n");
|
||||
}
|
||||
|
||||
@@ -418,13 +418,6 @@ pub enum BackendSummary {
|
||||
/// Configured key identifier
|
||||
key_id: String,
|
||||
},
|
||||
/// AWS KMS backend summary
|
||||
Aws {
|
||||
/// Configured region, when pinned instead of resolved by the AWS chain
|
||||
region: Option<String>,
|
||||
/// Endpoint override, when set for an emulator or private endpoint
|
||||
endpoint_url: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl From<&KmsConfig> for KmsConfigSummary {
|
||||
@@ -474,10 +467,6 @@ impl From<&KmsConfig> for KmsConfigSummary {
|
||||
BackendConfig::Static(static_config) => BackendSummary::Static {
|
||||
key_id: static_config.key_id.clone(),
|
||||
},
|
||||
BackendConfig::Aws(aws_config) => BackendSummary::Aws {
|
||||
region: aws_config.region.clone(),
|
||||
endpoint_url: aws_config.endpoint_url.clone(),
|
||||
},
|
||||
};
|
||||
|
||||
Self {
|
||||
|
||||
@@ -1,423 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Audit contract for KMS management operations.
|
||||
//!
|
||||
//! [`KmsManager`](crate::manager::KmsManager) builds a [`KmsAuditRecord`] for
|
||||
//! every management operation it serves and hands it to the installed
|
||||
//! [`KmsAuditSink`]. No sink is installed by default, so a deployment that
|
||||
//! does not consume KMS audit records pays nothing beyond the `Option` check.
|
||||
//!
|
||||
//! The record is deliberately a KMS-side type rather than an audit-crate
|
||||
//! entry: keeping the dependency edge out of this crate lets the server map
|
||||
//! records onto its existing audit pipeline (and its delivery semantics)
|
||||
//! without KMS having to know that pipeline exists.
|
||||
//!
|
||||
//! # Redaction
|
||||
//!
|
||||
//! A record has no field that can hold key material, and every value that
|
||||
//! originates from a caller passes through [`redact_encryption_context`]
|
||||
//! before it is stored. Adding a field here means re-checking that invariant.
|
||||
|
||||
use crate::error::KmsError;
|
||||
use crate::types::OperationContext;
|
||||
use rustfs_s3_types::EventName;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Encryption-context keys that describe *where* an object lives rather than
|
||||
/// anything secret about it. Their values are recorded verbatim; every other
|
||||
/// key is reduced to a digest.
|
||||
const ENCRYPTION_CONTEXT_ALLOWLIST: [&str; 5] = ["bucket", "object", "object_key", "algorithm", "sse_type"];
|
||||
|
||||
/// Prefix marking a value that was replaced by a digest of itself.
|
||||
const DIGEST_PREFIX: &str = "sha256:";
|
||||
|
||||
/// Hex characters of the digest that are kept. Enough to correlate repeated
|
||||
/// values across records without being reversible in practice.
|
||||
const DIGEST_LEN: usize = 16;
|
||||
|
||||
/// A KMS management operation that produces an audit record.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum KmsAuditOperation {
|
||||
/// Creation of a new master key.
|
||||
CreateKey,
|
||||
/// Metadata lookup for a single key.
|
||||
DescribeKey,
|
||||
/// Enumeration of keys.
|
||||
ListKeys,
|
||||
/// Scheduling a key for deletion after the pending window.
|
||||
ScheduleKeyDeletion,
|
||||
/// Cancelling a previously scheduled deletion.
|
||||
CancelKeyDeletion,
|
||||
/// Returning a disabled key to service.
|
||||
EnableKey,
|
||||
/// Taking a key out of service without destroying it.
|
||||
DisableKey,
|
||||
/// Rotating a key to a new version.
|
||||
RotateKey,
|
||||
/// Irreversible removal of key material once the pending window expired.
|
||||
DeleteKey,
|
||||
}
|
||||
|
||||
impl KmsAuditOperation {
|
||||
/// Stable operation name for audit consumers.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
KmsAuditOperation::CreateKey => "CreateKey",
|
||||
KmsAuditOperation::DescribeKey => "DescribeKey",
|
||||
KmsAuditOperation::ListKeys => "ListKeys",
|
||||
KmsAuditOperation::ScheduleKeyDeletion => "ScheduleKeyDeletion",
|
||||
KmsAuditOperation::CancelKeyDeletion => "CancelKeyDeletion",
|
||||
KmsAuditOperation::EnableKey => "EnableKey",
|
||||
KmsAuditOperation::DisableKey => "DisableKey",
|
||||
KmsAuditOperation::RotateKey => "RotateKey",
|
||||
KmsAuditOperation::DeleteKey => "DeleteKey",
|
||||
}
|
||||
}
|
||||
|
||||
/// Notification/audit event name carried by records for this operation.
|
||||
pub fn event_name(&self) -> EventName {
|
||||
match self {
|
||||
KmsAuditOperation::CreateKey => EventName::KmsKeyCreated,
|
||||
KmsAuditOperation::DescribeKey | KmsAuditOperation::ListKeys => EventName::KmsKeyAccessed,
|
||||
KmsAuditOperation::ScheduleKeyDeletion => EventName::KmsKeyDeletionScheduled,
|
||||
KmsAuditOperation::CancelKeyDeletion => EventName::KmsKeyDeletionCancelled,
|
||||
KmsAuditOperation::EnableKey => EventName::KmsKeyEnabled,
|
||||
KmsAuditOperation::DisableKey => EventName::KmsKeyDisabled,
|
||||
KmsAuditOperation::RotateKey => EventName::KmsKeyRotated,
|
||||
KmsAuditOperation::DeleteKey => EventName::KmsKeyDeleted,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the audited operation completed.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum KmsAuditOutcome {
|
||||
/// The operation completed and its effect is durable.
|
||||
Success,
|
||||
/// The operation failed; `error_class` carries the reason category.
|
||||
Failure,
|
||||
}
|
||||
|
||||
impl KmsAuditOutcome {
|
||||
/// Stable outcome name for audit consumers.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
KmsAuditOutcome::Success => "success",
|
||||
KmsAuditOutcome::Failure => "failure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Coarse, stable classification of a failed KMS operation.
|
||||
///
|
||||
/// Audit consumers alert on these, so the strings are a wire contract: add
|
||||
/// new classes rather than renaming existing ones.
|
||||
pub fn error_class(error: &KmsError) -> &'static str {
|
||||
match error {
|
||||
KmsError::ConfigurationError { .. } => "configuration",
|
||||
KmsError::KeyNotFound { .. } => "key_not_found",
|
||||
KmsError::InvalidKey { .. } => "invalid_key",
|
||||
KmsError::CryptographicError { .. } => "cryptographic",
|
||||
KmsError::BackendError { .. } => "backend",
|
||||
KmsError::AccessDenied { .. } => "access_denied",
|
||||
KmsError::KeyAlreadyExists { .. } => "key_already_exists",
|
||||
KmsError::InvalidOperation { .. } => "invalid_operation",
|
||||
KmsError::InternalError { .. } => "internal",
|
||||
KmsError::SerializationError { .. } => "serialization",
|
||||
KmsError::IoError { .. } => "io",
|
||||
KmsError::CacheError { .. } => "cache",
|
||||
KmsError::ValidationError { .. } => "validation",
|
||||
KmsError::UnsupportedAlgorithm { .. } => "unsupported_algorithm",
|
||||
KmsError::InvalidKeySize { .. } => "invalid_key_size",
|
||||
KmsError::ContextMismatch { .. } => "context_mismatch",
|
||||
KmsError::OperationTimedOut { .. } => "timeout",
|
||||
KmsError::OperationCancelled { .. } => "cancelled",
|
||||
KmsError::MaterialMissing { .. } => "material_missing",
|
||||
KmsError::MaterialCorrupt { .. } => "material_corrupt",
|
||||
KmsError::MaterialAuthenticationFailed { .. } => "material_authentication_failed",
|
||||
KmsError::UnsupportedFormatVersion { .. } => "unsupported_format_version",
|
||||
KmsError::KeyVersionNotFound { .. } => "key_version_not_found",
|
||||
KmsError::Backup(_) => "backup",
|
||||
KmsError::UnsupportedCapability { .. } => "unsupported_capability",
|
||||
KmsError::CredentialsUnavailable { .. } => "credentials_unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
/// One audited KMS management operation.
|
||||
///
|
||||
/// Everything an audit consumer needs to answer "who did what to which key,
|
||||
/// against which backend, and did it work" — and nothing that could carry key
|
||||
/// material. See the module docs for the redaction rules.
|
||||
///
|
||||
/// Tenant attribution is intentionally absent: multi-tenancy is not modelled
|
||||
/// yet, and an invented tenant value is worse than a missing one. It will
|
||||
/// arrive as an entry in [`Self::context`] once tenancy lands.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct KmsAuditRecord {
|
||||
/// Correlates every record emitted for one logical request.
|
||||
pub operation_id: Uuid,
|
||||
/// The audited operation.
|
||||
pub operation: KmsAuditOperation,
|
||||
/// Event name published to audit consumers.
|
||||
pub event: EventName,
|
||||
/// Authenticated identity that requested the operation, or
|
||||
/// [`OperationContext::INTERNAL_PRINCIPAL`] for server-initiated work.
|
||||
pub principal: String,
|
||||
/// Client address, when the caller supplied one.
|
||||
pub source_ip: Option<String>,
|
||||
/// Client user agent, when the caller supplied one.
|
||||
pub user_agent: Option<String>,
|
||||
/// Key the operation acted on. `None` for operations that span keys, such
|
||||
/// as listing.
|
||||
pub key_id: Option<String>,
|
||||
/// Key version the operation resolved to, when the result identifies one.
|
||||
/// Operations on a key as a whole leave this unset.
|
||||
pub key_version: Option<u32>,
|
||||
/// Whether the operation succeeded.
|
||||
pub outcome: KmsAuditOutcome,
|
||||
/// Failure category; `None` on success. See [`error_class`].
|
||||
pub error_class: Option<&'static str>,
|
||||
/// Backend that served the operation, e.g. `local` or `vault-transit`.
|
||||
pub backend: &'static str,
|
||||
/// Wall-clock time spent in the audited operation.
|
||||
pub latency: Duration,
|
||||
/// Retries performed above the audit point. `None` means the number is
|
||||
/// not observable here: backend-internal retries are accounted for by the
|
||||
/// `rustfs_kms_backend_operation_attempts` metric instead of being
|
||||
/// duplicated (and possibly contradicted) in the audit trail.
|
||||
pub retry_count: Option<u32>,
|
||||
/// Server-supplied correlation values carried by the operation context.
|
||||
/// Also the reserved slot for tenant attribution.
|
||||
pub context: BTreeMap<String, String>,
|
||||
/// Caller-supplied encryption context, after [`redact_encryption_context`].
|
||||
pub encryption_context: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl KmsAuditRecord {
|
||||
/// Start a record for `operation` performed under `context`.
|
||||
///
|
||||
/// The outcome defaults to failure so that a record which somehow escapes
|
||||
/// without [`Self::with_result`] under-reports success rather than
|
||||
/// inventing it.
|
||||
pub fn new(operation: KmsAuditOperation, context: &OperationContext, backend: &'static str) -> Self {
|
||||
Self {
|
||||
operation_id: context.operation_id,
|
||||
operation,
|
||||
event: operation.event_name(),
|
||||
principal: context.principal.clone(),
|
||||
source_ip: context.source_ip.clone(),
|
||||
user_agent: context.user_agent.clone(),
|
||||
key_id: None,
|
||||
key_version: None,
|
||||
outcome: KmsAuditOutcome::Failure,
|
||||
error_class: None,
|
||||
backend,
|
||||
latency: Duration::ZERO,
|
||||
retry_count: None,
|
||||
context: context
|
||||
.additional_context
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v.clone()))
|
||||
.collect(),
|
||||
encryption_context: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach the key the operation acted on.
|
||||
pub fn with_key_id(mut self, key_id: Option<impl Into<String>>) -> Self {
|
||||
self.key_id = key_id.map(Into::into);
|
||||
self
|
||||
}
|
||||
|
||||
/// Attach the key version the operation resolved to.
|
||||
pub fn with_key_version(mut self, key_version: Option<u32>) -> Self {
|
||||
self.key_version = key_version;
|
||||
self
|
||||
}
|
||||
|
||||
/// Attach the measured duration of the operation.
|
||||
pub fn with_latency(mut self, latency: Duration) -> Self {
|
||||
self.latency = latency;
|
||||
self
|
||||
}
|
||||
|
||||
/// Derive outcome and error class from the operation result.
|
||||
pub fn with_result<T>(mut self, result: &crate::error::Result<T>) -> Self {
|
||||
match result {
|
||||
Ok(_) => {
|
||||
self.outcome = KmsAuditOutcome::Success;
|
||||
self.error_class = None;
|
||||
}
|
||||
Err(error) => {
|
||||
self.outcome = KmsAuditOutcome::Failure;
|
||||
self.error_class = Some(error_class(error));
|
||||
}
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Attach the caller-supplied encryption context, redacted.
|
||||
///
|
||||
/// Redaction happens here rather than at the call site so no caller can
|
||||
/// place a raw context value into a record by accident.
|
||||
pub fn with_encryption_context(mut self, encryption_context: &HashMap<String, String>) -> Self {
|
||||
self.encryption_context = redact_encryption_context(encryption_context);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Reduce a caller-supplied encryption context to something safe to persist.
|
||||
///
|
||||
/// Keys naming the object's location are kept verbatim because they are the
|
||||
/// reason the context is audited at all. Every other value is replaced by a
|
||||
/// truncated digest, which still correlates repeated values across records
|
||||
/// but does not reproduce whatever the caller put there.
|
||||
pub fn redact_encryption_context(encryption_context: &HashMap<String, String>) -> BTreeMap<String, String> {
|
||||
encryption_context
|
||||
.iter()
|
||||
.map(|(key, value)| {
|
||||
let recorded = if ENCRYPTION_CONTEXT_ALLOWLIST.contains(&key.as_str()) {
|
||||
value.clone()
|
||||
} else {
|
||||
digest_value(value)
|
||||
};
|
||||
(key.clone(), recorded)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn digest_value(value: &str) -> String {
|
||||
let digest = hex::encode(Sha256::digest(value.as_bytes()));
|
||||
format!("{DIGEST_PREFIX}{}", &digest[..DIGEST_LEN])
|
||||
}
|
||||
|
||||
/// Receives KMS audit records.
|
||||
///
|
||||
/// Implementations must not block: they are called on the task that served
|
||||
/// the KMS operation. Delivery failures are the sink's problem — the KMS
|
||||
/// operation has already completed by the time a record is emitted, and its
|
||||
/// result is never changed by what the sink does.
|
||||
pub trait KmsAuditSink: Send + Sync {
|
||||
/// Handle one audit record.
|
||||
fn emit(&self, record: KmsAuditRecord);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn every_operation_maps_to_a_kms_event() {
|
||||
let operations = [
|
||||
KmsAuditOperation::CreateKey,
|
||||
KmsAuditOperation::DescribeKey,
|
||||
KmsAuditOperation::ListKeys,
|
||||
KmsAuditOperation::ScheduleKeyDeletion,
|
||||
KmsAuditOperation::CancelKeyDeletion,
|
||||
KmsAuditOperation::EnableKey,
|
||||
KmsAuditOperation::DisableKey,
|
||||
KmsAuditOperation::RotateKey,
|
||||
KmsAuditOperation::DeleteKey,
|
||||
];
|
||||
|
||||
for operation in operations {
|
||||
let event = operation.event_name();
|
||||
assert!(event.is_kms(), "{} must map to a KMS event, got {event}", operation.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_defaults_to_failure_until_a_result_is_attached() {
|
||||
let context = OperationContext::new("tester".to_string());
|
||||
let record = KmsAuditRecord::new(KmsAuditOperation::CreateKey, &context, "local");
|
||||
|
||||
assert_eq!(record.outcome, KmsAuditOutcome::Failure);
|
||||
assert!(record.error_class.is_none());
|
||||
|
||||
let ok: crate::error::Result<()> = Ok(());
|
||||
assert_eq!(record.clone().with_result(&ok).outcome, KmsAuditOutcome::Success);
|
||||
|
||||
let denied: crate::error::Result<()> = Err(KmsError::access_denied("nope"));
|
||||
let failed = record.with_result(&denied);
|
||||
assert_eq!(failed.outcome, KmsAuditOutcome::Failure);
|
||||
assert_eq!(failed.error_class, Some("access_denied"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_carries_the_full_operation_context() {
|
||||
let context = OperationContext::new("arn:aws:iam::user/alice".to_string())
|
||||
.with_source_ip("10.0.0.7".to_string())
|
||||
.with_user_agent("aws-cli/2".to_string())
|
||||
.with_context("requestID".to_string(), "req-1".to_string());
|
||||
|
||||
let record = KmsAuditRecord::new(KmsAuditOperation::RotateKey, &context, "vault-transit")
|
||||
.with_key_id(Some("key-1"))
|
||||
.with_key_version(Some(3))
|
||||
.with_latency(Duration::from_millis(12));
|
||||
|
||||
assert_eq!(record.operation_id, context.operation_id);
|
||||
assert_eq!(record.principal, "arn:aws:iam::user/alice");
|
||||
assert_eq!(record.source_ip.as_deref(), Some("10.0.0.7"));
|
||||
assert_eq!(record.user_agent.as_deref(), Some("aws-cli/2"));
|
||||
assert_eq!(record.key_id.as_deref(), Some("key-1"));
|
||||
assert_eq!(record.key_version, Some(3));
|
||||
assert_eq!(record.backend, "vault-transit");
|
||||
assert_eq!(record.latency, Duration::from_millis(12));
|
||||
assert_eq!(record.event, EventName::KmsKeyRotated);
|
||||
assert_eq!(record.context.get("requestID").map(String::as_str), Some("req-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encryption_context_keeps_only_allowlisted_values_verbatim() {
|
||||
let secret = "eyJhbGciOiJIUzI1NiJ9.super-secret-grant-token";
|
||||
let context = HashMap::from([
|
||||
("bucket".to_string(), "photos".to_string()),
|
||||
("object_key".to_string(), "2026/cat.png".to_string()),
|
||||
("algorithm".to_string(), "AES256".to_string()),
|
||||
("grant_token".to_string(), secret.to_string()),
|
||||
("customer_reference".to_string(), "acct-4711".to_string()),
|
||||
]);
|
||||
|
||||
let redacted = redact_encryption_context(&context);
|
||||
|
||||
assert_eq!(redacted.get("bucket").map(String::as_str), Some("photos"));
|
||||
assert_eq!(redacted.get("object_key").map(String::as_str), Some("2026/cat.png"));
|
||||
assert_eq!(redacted.get("algorithm").map(String::as_str), Some("AES256"));
|
||||
|
||||
for key in ["grant_token", "customer_reference"] {
|
||||
let value = redacted.get(key).expect("non-allowlisted key should be kept as a digest");
|
||||
assert!(value.starts_with(DIGEST_PREFIX), "{key} should be digested, got {value}");
|
||||
}
|
||||
|
||||
let rendered = format!("{redacted:?}");
|
||||
assert!(!rendered.contains(secret), "redacted context must not reproduce the raw value");
|
||||
assert!(!rendered.contains("acct-4711"), "redacted context must not reproduce the raw value");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_values_digest_identically_across_records() {
|
||||
// Correlation is the reason a digest is preferable to a fixed
|
||||
// placeholder; assert it actually holds.
|
||||
let first = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
|
||||
let second = redact_encryption_context(&HashMap::from([("tenant".to_string(), "alpha".to_string())]));
|
||||
let other = redact_encryption_context(&HashMap::from([("tenant".to_string(), "beta".to_string())]));
|
||||
|
||||
assert_eq!(first.get("tenant"), second.get("tenant"));
|
||||
assert_ne!(first.get("tenant"), other.get("tenant"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,10 +14,7 @@
|
||||
|
||||
//! Local file-based KMS backend implementation
|
||||
|
||||
use crate::backends::{
|
||||
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits,
|
||||
ensure_tag_keys_are_mutable,
|
||||
};
|
||||
use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_status_permits};
|
||||
use crate::config::KmsConfig;
|
||||
use crate::config::LocalConfig;
|
||||
use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
|
||||
@@ -1297,32 +1294,6 @@ impl LocalKmsClient {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read-modify-write of a key's mutable metadata under the per-key write
|
||||
/// lock, carrying the existing key material over untouched.
|
||||
///
|
||||
/// `mutate` reports whether it changed anything; an unchanged record is
|
||||
/// never rewritten, so a repeated no-op update neither rewrites the key
|
||||
/// file nor risks a failed commit on an unrelated call.
|
||||
async fn update_key_metadata<F>(&self, key_id: &str, mutate: F) -> Result<()>
|
||||
where
|
||||
F: FnOnce(&mut MasterKeyInfo) -> Result<bool>,
|
||||
{
|
||||
let _write_guard = self.lock_key_for_write(key_id).await;
|
||||
let mut master_key = self.load_master_key(key_id).await?;
|
||||
if !mutate(&mut master_key)? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Preserve the existing key material (see enable_key): a metadata edit
|
||||
// must never regenerate the master key, or every DEK wrapped by it
|
||||
// becomes undecryptable.
|
||||
let key_material = self.get_key_material(key_id).await?;
|
||||
self.save_master_key(&master_key, &key_material).await?;
|
||||
|
||||
debug!(key_id, "Local KMS key metadata updated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn schedule_key_deletion(
|
||||
@@ -1415,8 +1386,7 @@ impl LocalKmsBackend {
|
||||
crate::config::BackendConfig::Local(local_config) => local_config.clone(),
|
||||
crate::config::BackendConfig::VaultKv2(_)
|
||||
| crate::config::BackendConfig::VaultTransit(_)
|
||||
| crate::config::BackendConfig::Static(_)
|
||||
| crate::config::BackendConfig::Aws(_) => {
|
||||
| crate::config::BackendConfig::Static(_) => {
|
||||
return Err(KmsError::configuration_error("Expected Local backend configuration"));
|
||||
}
|
||||
};
|
||||
@@ -1441,15 +1411,12 @@ impl KmsBackend for LocalKmsBackend {
|
||||
// Generate key material
|
||||
let key_material = generate_key_material(algorithm)?;
|
||||
|
||||
let mut master_key = MasterKeyInfo::new_with_description(
|
||||
let master_key = MasterKeyInfo::new_with_description(
|
||||
key_id.clone(),
|
||||
algorithm.to_string(),
|
||||
Some("local-kms".to_string()),
|
||||
request.description.clone(),
|
||||
);
|
||||
// Persist the caller's tags: the response below reports them, and
|
||||
// describe_key reads them back out of this record.
|
||||
master_key.metadata = request.tags.clone();
|
||||
|
||||
// Save to disk
|
||||
self.client.save_new_master_key(&master_key, &key_material).await?;
|
||||
@@ -1717,44 +1684,6 @@ impl KmsBackend for LocalKmsBackend {
|
||||
self.client.disable_key(key_id, None).await
|
||||
}
|
||||
|
||||
async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.client
|
||||
.update_key_metadata(key_id, |master_key| {
|
||||
if master_key.description.as_deref() == description {
|
||||
return Ok(false);
|
||||
}
|
||||
master_key.description = description.map(str::to_string);
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
|
||||
self.client
|
||||
.update_key_metadata(key_id, |master_key| {
|
||||
let mut changed = false;
|
||||
for (tag_key, value) in tags {
|
||||
changed |= master_key.metadata.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
|
||||
}
|
||||
Ok(changed)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
|
||||
self.client
|
||||
.update_key_metadata(key_id, |master_key| {
|
||||
let mut changed = false;
|
||||
for tag_key in tag_keys {
|
||||
changed |= master_key.metadata.remove(tag_key).is_some();
|
||||
}
|
||||
Ok(changed)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<bool> {
|
||||
self.client.health_check().await.map(|_| true)
|
||||
}
|
||||
@@ -1767,7 +1696,6 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.with_enable_disable(true)
|
||||
.with_schedule_deletion(true)
|
||||
.with_physical_delete(true)
|
||||
.with_update_key_metadata(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
|
||||
@@ -19,9 +19,7 @@ use crate::types::*;
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub mod aws;
|
||||
#[cfg(test)]
|
||||
mod contract_tests;
|
||||
pub mod local;
|
||||
@@ -100,32 +98,6 @@ pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operat
|
||||
ensure_key_state_permits(key_id, &state, operation)
|
||||
}
|
||||
|
||||
/// Tag key that carries the key's identity rather than user metadata.
|
||||
///
|
||||
/// The key-creation path lifts `name` out of the caller's tag map and uses it
|
||||
/// as the key id, so a key's `name` tag and its id are the same string.
|
||||
/// Rewriting or dropping it after creation would leave the key addressable
|
||||
/// under an id its own metadata no longer states. Metadata updates therefore
|
||||
/// reject it; only creation may set it.
|
||||
pub const RESERVED_KEY_NAME_TAG: &str = "name";
|
||||
|
||||
/// Reject a metadata update that would rewrite or remove
|
||||
/// [`RESERVED_KEY_NAME_TAG`].
|
||||
///
|
||||
/// Enforced by every backend that implements tag updates, so no call path —
|
||||
/// including a direct [`KmsBackend`] user that bypasses the manager — can
|
||||
/// detach a key from its identity.
|
||||
pub(crate) fn ensure_tag_keys_are_mutable<'a>(tag_keys: impl IntoIterator<Item = &'a str>) -> Result<()> {
|
||||
for tag_key in tag_keys {
|
||||
if tag_key == RESERVED_KEY_NAME_TAG {
|
||||
return Err(KmsError::invalid_parameter(format!(
|
||||
"Tag '{RESERVED_KEY_NAME_TAG}' identifies the key and cannot be updated or removed"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Simplified KMS backend interface for manager
|
||||
#[async_trait]
|
||||
pub trait KmsBackend: Send + Sync {
|
||||
@@ -181,38 +153,6 @@ pub trait KmsBackend: Send + Sync {
|
||||
Err(KmsError::unsupported_capability("backend without rotation support", "rotate_key"))
|
||||
}
|
||||
|
||||
/// Replace a key's free-form description; `None` clears it.
|
||||
///
|
||||
/// Backends that advertise [`BackendCapabilities::update_key_metadata`]
|
||||
/// must override this method; the default rejects the operation.
|
||||
async fn update_key_description(&self, _key_id: &str, _description: Option<&str>) -> Result<()> {
|
||||
Err(KmsError::unsupported_capability(
|
||||
"backend without key metadata updates",
|
||||
"update_key_description",
|
||||
))
|
||||
}
|
||||
|
||||
/// Add or overwrite the given tags, leaving every other tag untouched.
|
||||
///
|
||||
/// Implementations must run [`ensure_tag_keys_are_mutable`] before
|
||||
/// persisting anything. Backends that advertise
|
||||
/// [`BackendCapabilities::update_key_metadata`] must override this method;
|
||||
/// the default rejects the operation.
|
||||
async fn tag_key(&self, _key_id: &str, _tags: &HashMap<String, String>) -> Result<()> {
|
||||
Err(KmsError::unsupported_capability("backend without key metadata updates", "tag_key"))
|
||||
}
|
||||
|
||||
/// Remove the given tags.
|
||||
///
|
||||
/// Tags that are not set are ignored, so repeating the call is a no-op
|
||||
/// rather than an error. Implementations must run
|
||||
/// [`ensure_tag_keys_are_mutable`] before persisting anything. Backends
|
||||
/// that advertise [`BackendCapabilities::update_key_metadata`] must
|
||||
/// override this method; the default rejects the operation.
|
||||
async fn untag_key(&self, _key_id: &str, _tag_keys: &[String]) -> Result<()> {
|
||||
Err(KmsError::unsupported_capability("backend without key metadata updates", "untag_key"))
|
||||
}
|
||||
|
||||
/// Health check
|
||||
async fn health_check(&self) -> Result<bool>;
|
||||
|
||||
@@ -282,8 +222,6 @@ pub struct BackendCapabilities {
|
||||
pub versioning: bool,
|
||||
/// Irreversible physical deletion of key material
|
||||
pub physical_delete: bool,
|
||||
/// Updating a key's description and tags after creation
|
||||
pub update_key_metadata: bool,
|
||||
}
|
||||
|
||||
impl BackendCapabilities {
|
||||
@@ -300,7 +238,6 @@ impl BackendCapabilities {
|
||||
schedule_deletion: false,
|
||||
versioning: false,
|
||||
physical_delete: false,
|
||||
update_key_metadata: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -351,12 +288,6 @@ impl BackendCapabilities {
|
||||
self.physical_delete = physical_delete;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether description and tag updates are supported
|
||||
pub const fn with_update_key_metadata(mut self, update_key_metadata: bool) -> Self {
|
||||
self.update_key_metadata = update_key_metadata;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BackendCapabilities {
|
||||
@@ -435,7 +366,6 @@ mod tests {
|
||||
assert!(!capabilities.schedule_deletion);
|
||||
assert!(!capabilities.versioning);
|
||||
assert!(!capabilities.physical_delete);
|
||||
assert!(!capabilities.update_key_metadata);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -444,12 +374,6 @@ mod tests {
|
||||
("enable_key", MinimalBackend.enable_key("any-key").await),
|
||||
("disable_key", MinimalBackend.disable_key("any-key").await),
|
||||
("rotate_key", MinimalBackend.rotate_key("any-key").await),
|
||||
(
|
||||
"update_key_description",
|
||||
MinimalBackend.update_key_description("any-key", Some("new")).await,
|
||||
),
|
||||
("tag_key", MinimalBackend.tag_key("any-key", &HashMap::new()).await),
|
||||
("untag_key", MinimalBackend.untag_key("any-key", &[]).await),
|
||||
] {
|
||||
let error = result.expect_err("backends must opt in to lifecycle operations by overriding them");
|
||||
assert!(
|
||||
@@ -471,20 +395,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_tag_is_rejected_by_metadata_updates() {
|
||||
let error = ensure_tag_keys_are_mutable([RESERVED_KEY_NAME_TAG])
|
||||
.expect_err("the identity tag must not be writable through a metadata update");
|
||||
assert!(
|
||||
matches!(&error, KmsError::InvalidOperation { message } if message.contains(RESERVED_KEY_NAME_TAG)),
|
||||
"expected a typed rejection naming the tag, got {error:?}"
|
||||
);
|
||||
|
||||
// Ordinary tags — including ones that merely contain the reserved name
|
||||
// — stay writable.
|
||||
ensure_tag_keys_are_mutable(["team", "nickname", "Name"]).expect("ordinary tags must remain writable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_backend_capabilities_golden() {
|
||||
let temp_dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
---
|
||||
source: crates/kms/src/backends/aws.rs
|
||||
expression: capabilities_snapshot(backend.capabilities())
|
||||
---
|
||||
{
|
||||
"decrypt": true,
|
||||
"enable_disable": true,
|
||||
"encrypt": true,
|
||||
"generate_data_key": true,
|
||||
"physical_delete": false,
|
||||
"rotate": true,
|
||||
"schedule_deletion": true,
|
||||
"update_key_metadata": false,
|
||||
"versioning": true
|
||||
}
|
||||
-1
@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
|
||||
"physical_delete": true,
|
||||
"rotate": false,
|
||||
"schedule_deletion": true,
|
||||
"update_key_metadata": true,
|
||||
"versioning": false
|
||||
}
|
||||
|
||||
-1
@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
|
||||
"physical_delete": false,
|
||||
"rotate": false,
|
||||
"schedule_deletion": false,
|
||||
"update_key_metadata": false,
|
||||
"versioning": false
|
||||
}
|
||||
|
||||
-1
@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
|
||||
"physical_delete": true,
|
||||
"rotate": true,
|
||||
"schedule_deletion": true,
|
||||
"update_key_metadata": true,
|
||||
"versioning": true
|
||||
}
|
||||
|
||||
-1
@@ -10,6 +10,5 @@ expression: capabilities_snapshot(backend.capabilities())
|
||||
"physical_delete": true,
|
||||
"rotate": true,
|
||||
"schedule_deletion": true,
|
||||
"update_key_metadata": true,
|
||||
"versioning": true
|
||||
}
|
||||
|
||||
@@ -431,32 +431,6 @@ mod tests {
|
||||
(backend, key_id, key)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_updates_report_the_capability_gap() {
|
||||
let (backend, key_id, _key) = create_test_backend().await;
|
||||
assert!(
|
||||
!backend.capabilities().update_key_metadata,
|
||||
"Static KMS owns no mutable key record, so it must not advertise metadata updates"
|
||||
);
|
||||
|
||||
for (operation, result) in [
|
||||
("update_key_description", backend.update_key_description(&key_id, Some("new")).await),
|
||||
(
|
||||
"tag_key",
|
||||
backend
|
||||
.tag_key(&key_id, &HashMap::from([("team".to_string(), "storage".to_string())]))
|
||||
.await,
|
||||
),
|
||||
("untag_key", backend.untag_key(&key_id, &["team".to_string()]).await),
|
||||
] {
|
||||
let error = result.expect_err("Static KMS is read-only: metadata updates must be rejected");
|
||||
assert!(
|
||||
matches!(error, KmsError::UnsupportedCapability { .. }),
|
||||
"expected UnsupportedCapability for {operation}, got {error:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_generate_and_decrypt_data_key() {
|
||||
let (backend, key_id, _key) = create_test_backend().await;
|
||||
|
||||
@@ -20,7 +20,6 @@ use crate::backends::vault_credentials::{
|
||||
};
|
||||
use crate::backends::{
|
||||
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits, ensure_key_status_permits,
|
||||
ensure_tag_keys_are_mutable,
|
||||
};
|
||||
use crate::config::{KmsConfig, VaultConfig};
|
||||
use crate::encryption::{AesDekCrypto, DataKeyEnvelope, DekCrypto, generate_key_material};
|
||||
@@ -1021,67 +1020,6 @@ impl VaultKmsClient {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Replace the key's description; `None` clears it.
|
||||
///
|
||||
/// The write goes through the check-and-set read-modify-write loop, so a
|
||||
/// rotation or state transition landing in between is carried over instead
|
||||
/// of clobbered. A description that already matches is not rewritten.
|
||||
pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.update_key_data_with_cas(key_id, |key_data| {
|
||||
if key_data.description.as_deref() == description {
|
||||
return Ok(CasMutation::Skip(()));
|
||||
}
|
||||
key_data.description = description.map(str::to_string);
|
||||
Ok(CasMutation::Write(()))
|
||||
})
|
||||
.await?;
|
||||
|
||||
debug!(key_id, "Vault KMS key description updated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add or overwrite tags, leaving every other tag untouched.
|
||||
pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
|
||||
|
||||
self.update_key_data_with_cas(key_id, |key_data| {
|
||||
let mut changed = false;
|
||||
for (tag_key, value) in tags {
|
||||
changed |= key_data.tags.insert(tag_key.clone(), value.clone()).as_ref() != Some(value);
|
||||
}
|
||||
Ok(if changed {
|
||||
CasMutation::Write(())
|
||||
} else {
|
||||
CasMutation::Skip(())
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
|
||||
debug!(key_id, "Vault KMS key tags updated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove tags; tags that are not set are ignored.
|
||||
pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
|
||||
|
||||
self.update_key_data_with_cas(key_id, |key_data| {
|
||||
let mut changed = false;
|
||||
for tag_key in tag_keys {
|
||||
changed |= key_data.tags.remove(tag_key).is_some();
|
||||
}
|
||||
Ok(if changed {
|
||||
CasMutation::Write(())
|
||||
} else {
|
||||
CasMutation::Skip(())
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
|
||||
debug!(key_id, "Vault KMS key tags removed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rotate the master key while keeping every historical version decryptable.
|
||||
///
|
||||
/// Commit protocol (all writes check-and-set, in this order):
|
||||
@@ -1225,8 +1163,7 @@ impl VaultKmsBackend {
|
||||
crate::config::BackendConfig::VaultKv2(vault_config) => (**vault_config).clone(),
|
||||
crate::config::BackendConfig::Local(_)
|
||||
| crate::config::BackendConfig::VaultTransit(_)
|
||||
| crate::config::BackendConfig::Static(_)
|
||||
| crate::config::BackendConfig::Aws(_) => {
|
||||
| crate::config::BackendConfig::Static(_) => {
|
||||
return Err(KmsError::configuration_error("Expected Vault KV2 backend configuration"));
|
||||
}
|
||||
};
|
||||
@@ -1509,18 +1446,6 @@ impl KmsBackend for VaultKmsBackend {
|
||||
self.client.rotate_key(key_id, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.client.update_key_description(key_id, description).await
|
||||
}
|
||||
|
||||
async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
self.client.tag_key(key_id, tags).await
|
||||
}
|
||||
|
||||
async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
self.client.untag_key(key_id, tag_keys).await
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<bool> {
|
||||
self.client.health_check().await.map(|_| true)
|
||||
}
|
||||
@@ -1536,7 +1461,6 @@ impl KmsBackend for VaultKmsBackend {
|
||||
.with_schedule_deletion(true)
|
||||
.with_versioning(true)
|
||||
.with_physical_delete(true)
|
||||
.with_update_key_metadata(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
@@ -2920,63 +2844,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Tag updates are check-and-set read-modify-writes over the live record,
|
||||
/// never blind overwrites: they preserve the material and the tags they did
|
||||
/// not address.
|
||||
#[tokio::test]
|
||||
async fn wired_tag_key_writeback_is_check_and_set() {
|
||||
let mut key_data = healthy_key_data();
|
||||
key_data.tags = HashMap::from([("name".to_string(), "wired-key".to_string())]);
|
||||
let (vault, client) = scripted_client(vec![
|
||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
||||
ScriptedResponse::ok(kv2_read_data(&key_data)),
|
||||
ScriptedResponse::ok(kv2_write_ack()),
|
||||
])
|
||||
.await;
|
||||
|
||||
client
|
||||
.tag_key("wired-key", &HashMap::from([("team".to_string(), "storage".to_string())]))
|
||||
.await
|
||||
.expect("tagging must succeed");
|
||||
|
||||
let bodies = vault.request_bodies();
|
||||
let writeback = parse_write_body(&bodies[2]);
|
||||
assert_eq!(writeback["options"]["cas"], serde_json::json!(1), "{writeback}");
|
||||
assert_eq!(writeback["data"]["tags"]["team"], serde_json::json!("storage"), "{writeback}");
|
||||
assert_eq!(
|
||||
writeback["data"]["tags"]["name"],
|
||||
serde_json::json!("wired-key"),
|
||||
"a tag update must not drop tags it did not address: {writeback}"
|
||||
);
|
||||
assert_eq!(
|
||||
writeback["data"]["encrypted_key_material"],
|
||||
serde_json::json!(healthy_key_data().encrypted_key_material),
|
||||
"the write-back must preserve the material of the freshly read record: {writeback}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Rejecting the identity tag happens before any Vault call, so a rejected
|
||||
/// request cannot leave a partial write behind.
|
||||
#[tokio::test]
|
||||
async fn wired_identity_tag_update_is_rejected_before_any_vault_call() {
|
||||
let (vault, client) = scripted_client(Vec::new()).await;
|
||||
|
||||
for result in [
|
||||
client
|
||||
.tag_key("wired-key", &HashMap::from([("name".to_string(), "other".to_string())]))
|
||||
.await,
|
||||
client.untag_key("wired-key", &["name".to_string()]).await,
|
||||
] {
|
||||
let error = result.expect_err("the identity tag must not be writable");
|
||||
assert!(matches!(error, KmsError::InvalidOperation { .. }), "got {error:?}");
|
||||
}
|
||||
assert!(
|
||||
vault.request_bodies().is_empty(),
|
||||
"a rejected metadata update must not reach Vault: {:?}",
|
||||
vault.request_bodies()
|
||||
);
|
||||
}
|
||||
|
||||
/// A version record above the current pointer means the top-level record
|
||||
/// regressed (a lost update rolled back a committed rotation). Resolving
|
||||
/// material through such a record must fail closed instead of quietly
|
||||
|
||||
@@ -57,42 +57,6 @@ const DEFAULT_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(5);
|
||||
/// Default seconds between token file re-reads for [`TokenFileSource`].
|
||||
const DEFAULT_TOKEN_FILE_POLL_INTERVAL_SECS: u64 = 30;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metrics
|
||||
//
|
||||
// Both gauges describe the one credential generation currently installed, so
|
||||
// they carry no labels: the Vault address, mount, auth path and token are all
|
||||
// off limits as label values, and there is exactly one generation to describe.
|
||||
// The renewal loop republishes them on a bounded cadence while it waits, so a
|
||||
// scrape landing between refresh cycles never reads a TTL frozen at the last
|
||||
// refresh, or a fail-closed state that flipped after it.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Gauge: seconds left before the active Vault token expires; `0` once it has.
|
||||
const METRIC_TOKEN_TTL_SECONDS: &str = "rustfs_kms_vault_token_ttl_seconds";
|
||||
/// Gauge: `1` while [`VaultCredentialProvider::current`] refuses to hand out
|
||||
/// the token because it is inside the fail-closed safety window, `0` otherwise.
|
||||
const METRIC_CREDENTIALS_FAIL_CLOSED: &str = "rustfs_kms_vault_credentials_fail_closed";
|
||||
|
||||
/// How often the renewal loop republishes the credential gauges while waiting.
|
||||
/// Bounds how stale a scrape can be, without any additional Vault traffic.
|
||||
const CREDENTIAL_GAUGE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
|
||||
/// Register metric descriptions once per process.
|
||||
fn describe_credential_metrics() {
|
||||
static DESCRIBE: std::sync::Once = std::sync::Once::new();
|
||||
DESCRIBE.call_once(|| {
|
||||
metrics::describe_gauge!(
|
||||
METRIC_TOKEN_TTL_SECONDS,
|
||||
"Seconds remaining before the Vault token backing the KMS backend expires"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
METRIC_CREDENTIALS_FAIL_CLOSED,
|
||||
"1 while the Vault credential provider refuses to serve its token because the token is inside the fail-closed safety window"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// A crate-owned secret value, zeroized on drop and redacted in Debug output.
|
||||
#[derive(Clone, Zeroize, ZeroizeOnDrop)]
|
||||
pub(crate) struct SecretString(String);
|
||||
@@ -663,45 +627,6 @@ impl VaultCredentialProvider {
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
/// Publish the credential gauges for the generation currently installed.
|
||||
///
|
||||
/// The fail-closed gauge re-evaluates the very gate
|
||||
/// [`VaultCredentialProvider::current`] applies, so what operators see and
|
||||
/// what the request path does cannot drift apart.
|
||||
fn record_credential_gauges(&self) {
|
||||
let handle = self.current.load();
|
||||
let now = Instant::now();
|
||||
let fail_closed = match handle.expires_at() {
|
||||
Some(expires_at) => {
|
||||
metrics::gauge!(METRIC_TOKEN_TTL_SECONDS).set(expires_at.saturating_duration_since(now).as_secs_f64());
|
||||
now + self.policy.safety_window >= expires_at
|
||||
}
|
||||
// A generation without an expiry has no remaining TTL to report
|
||||
// and can never lapse, so it can never fail closed either.
|
||||
None => false,
|
||||
};
|
||||
metrics::gauge!(METRIC_CREDENTIALS_FAIL_CLOSED).set(if fail_closed { 1.0 } else { 0.0 });
|
||||
}
|
||||
|
||||
/// Wait until `deadline`, republishing the credential gauges on the
|
||||
/// observation cadence. Reports `false` when cancellation cut the wait
|
||||
/// short.
|
||||
async fn wait_publishing_gauges(&self, deadline: Instant, cancel: &CancellationToken) -> bool {
|
||||
loop {
|
||||
self.record_credential_gauges();
|
||||
let now = Instant::now();
|
||||
if now >= deadline {
|
||||
return true;
|
||||
}
|
||||
let slice = (deadline - now).min(CREDENTIAL_GAUGE_INTERVAL);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return false,
|
||||
_ = tokio::time::sleep(slice) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Refresh the credentials if generation `observed` is still current.
|
||||
///
|
||||
/// Single-flight: concurrent callers serialize on the refresh lock, and a
|
||||
@@ -786,22 +711,17 @@ impl fmt::Debug for VaultCredentialProvider {
|
||||
/// again immediately (the renewal point is already in the past), so the
|
||||
/// provider keeps trying to recover even after the fail-closed window has
|
||||
/// been reached.
|
||||
///
|
||||
/// Both waits run through [`VaultCredentialProvider::wait_publishing_gauges`],
|
||||
/// which is the only place the credential gauges are published: the loop is
|
||||
/// already the component that tracks token expiry, and doing it here keeps the
|
||||
/// request path free of any metric work.
|
||||
async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: CancellationToken) {
|
||||
describe_credential_metrics();
|
||||
loop {
|
||||
let handle = provider.snapshot();
|
||||
let Some(renew_at) = handle.renew_at() else {
|
||||
// The current generation never expires; nothing left to schedule.
|
||||
provider.record_credential_gauges();
|
||||
return;
|
||||
};
|
||||
if !provider.wait_publishing_gauges(renew_at, &cancel).await {
|
||||
return;
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return,
|
||||
_ = tokio::time::sleep_until(renew_at) => {}
|
||||
}
|
||||
|
||||
match provider.refresh(handle.generation, &cancel).await {
|
||||
@@ -813,9 +733,10 @@ async fn renewal_loop(provider: Arc<VaultCredentialProvider>, cancel: Cancellati
|
||||
error = %error,
|
||||
"Vault credential refresh failed; retrying until the credentials recover"
|
||||
);
|
||||
let retry_at = Instant::now() + provider.policy.retry_interval;
|
||||
if !provider.wait_publishing_gauges(retry_at, &cancel).await {
|
||||
return;
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => return,
|
||||
_ = tokio::time::sleep(provider.policy.retry_interval) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1433,125 +1354,4 @@ mod tests {
|
||||
assert!(!rendered.contains(TEST_TOKEN), "debug output must not leak the vault token: {rendered}");
|
||||
assert!(rendered.contains("vault-token"), "the file path is not a secret");
|
||||
}
|
||||
|
||||
// -- Metric emission ----------------------------------------------------
|
||||
//
|
||||
// Each test installs a thread-local debugging recorder and drives a
|
||||
// paused-clock current-thread runtime inside it, so the renewal loop's
|
||||
// gauge publications land on exact virtual timestamps.
|
||||
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
|
||||
type MetricEntry = (
|
||||
metrics_util::CompositeKey,
|
||||
Option<metrics::Unit>,
|
||||
Option<metrics::SharedString>,
|
||||
DebugValue,
|
||||
);
|
||||
|
||||
/// Run `test` on a paused current-thread runtime under a debugging
|
||||
/// recorder and return one snapshot of everything it emitted.
|
||||
///
|
||||
/// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
|
||||
/// the recorded state, so taking it per assertion would only show the
|
||||
/// first assertion any data.
|
||||
fn record_metrics<Out>(
|
||||
test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
|
||||
) -> (Vec<MetricEntry>, Out) {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let out = metrics::with_local_recorder(&recorder, || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_time()
|
||||
.start_paused(true)
|
||||
.build()
|
||||
.expect("current-thread runtime must build");
|
||||
runtime.block_on(test())
|
||||
});
|
||||
(snapshotter.snapshot().into_vec(), out)
|
||||
}
|
||||
|
||||
/// Last value of a gauge, plus the labels it was published with.
|
||||
fn gauge(snapshot: &[MetricEntry], name: &str) -> Option<(f64, Vec<String>)> {
|
||||
snapshot.iter().find_map(|(composite, _unit, _description, value)| {
|
||||
let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Gauge(value)) => Some((
|
||||
value.into_inner(),
|
||||
composite.key().labels().map(|label| label.key().to_string()).collect(),
|
||||
)),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn renewal_loop_republishes_token_ttl_while_it_waits() {
|
||||
let (snapshot, ()) = record_metrics(|| {
|
||||
Box::pin(async {
|
||||
let (provider, _state) = scripted_provider(
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
test_policy(Duration::from_secs(10), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
let task = provider.spawn_renewal_task().expect("lease-bound tokens need renewal");
|
||||
|
||||
// Well inside the first half of the lease: nothing has been
|
||||
// renewed yet, so only the observation cadence can have moved
|
||||
// the gauge off its initial 60s.
|
||||
tokio::time::sleep(Duration::from_secs(25)).await;
|
||||
task.shutdown().await;
|
||||
})
|
||||
});
|
||||
|
||||
let (ttl, labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
|
||||
assert!(
|
||||
(ttl - 40.0).abs() < 1.0,
|
||||
"expected ~40s left of a 60s lease at the last observation, got {ttl}"
|
||||
);
|
||||
assert!(labels.is_empty(), "credential gauges must carry no labels, got {labels:?}");
|
||||
assert_eq!(
|
||||
gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
|
||||
Some(0.0),
|
||||
"a token outside its safety window must not report fail-closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_closed_gauge_tracks_the_gate_the_request_path_applies() {
|
||||
let (snapshot, refused) = record_metrics(|| {
|
||||
Box::pin(async {
|
||||
let (provider, state) = scripted_provider(
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
test_policy(Duration::from_secs(10), Duration::from_secs(5)),
|
||||
)
|
||||
.await;
|
||||
state.fail_renew.store(true, Ordering::SeqCst);
|
||||
state.fail_login.store(true, Ordering::SeqCst);
|
||||
let task = provider.spawn_renewal_task().expect("renewal task");
|
||||
|
||||
// 60s lease minus the 10s safety window: by 51s the provider
|
||||
// is refusing the token, and the gauge must already say so.
|
||||
tokio::time::sleep(Duration::from_secs(51)).await;
|
||||
let refused = provider.current().is_err();
|
||||
task.shutdown().await;
|
||||
refused
|
||||
})
|
||||
});
|
||||
|
||||
assert!(refused, "a token inside the safety window must be refused");
|
||||
assert_eq!(
|
||||
gauge(&snapshot, METRIC_CREDENTIALS_FAIL_CLOSED).map(|(value, _)| value),
|
||||
Some(1.0),
|
||||
"the gauge must report the same fail-closed state the request path enforces"
|
||||
);
|
||||
let (ttl, _labels) = gauge(&snapshot, METRIC_TOKEN_TTL_SECONDS).expect("token TTL gauge must be published");
|
||||
assert!(
|
||||
(ttl - 10.0).abs() < 1.0,
|
||||
"expected the TTL gauge to track the lease down into its safety window, got {ttl}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,10 +18,7 @@ use crate::backends::vault_credentials::{
|
||||
CredentialTaskHandle, VaultClientHandle, VaultConnectionSettings, VaultCredentialPolicy, VaultCredentialProvider,
|
||||
token_source_for,
|
||||
};
|
||||
use crate::backends::{
|
||||
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits,
|
||||
ensure_tag_keys_are_mutable,
|
||||
};
|
||||
use crate::backends::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, StateGatedOperation, ensure_key_state_permits};
|
||||
use crate::config::{KmsConfig, VaultTransitConfig};
|
||||
use crate::encryption::{DataKeyEnvelope, generate_key_material};
|
||||
use crate::error::{KmsError, Result};
|
||||
@@ -911,46 +908,6 @@ impl VaultTransitKmsClient {
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Replace the key's description; `None` clears it.
|
||||
///
|
||||
/// Metadata edits carry no state gate: they neither use nor invalidate key
|
||||
/// material, so they stay available for whatever lifecycle state the key
|
||||
/// is in.
|
||||
pub(crate) async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.mutate_key_metadata(key_id, |metadata| {
|
||||
metadata.description = description.map(str::to_string);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Add or overwrite tags, leaving every other tag untouched.
|
||||
pub(crate) async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tags.keys().map(String::as_str))?;
|
||||
self.mutate_key_metadata(key_id, |metadata| {
|
||||
metadata
|
||||
.tags
|
||||
.extend(tags.iter().map(|(key, value)| (key.clone(), value.clone())));
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Remove tags; tags that are not set are ignored.
|
||||
pub(crate) async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
ensure_tag_keys_are_mutable(tag_keys.iter().map(String::as_str))?;
|
||||
self.mutate_key_metadata(key_id, |metadata| {
|
||||
for tag_key in tag_keys {
|
||||
metadata.tags.remove(tag_key);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// Test-only lifecycle driver: the product path goes through [`KmsBackend`].
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn schedule_key_deletion(
|
||||
@@ -1055,9 +1012,7 @@ impl VaultTransitKmsBackend {
|
||||
metadata_key_prefix: vault_config.key_path_prefix.clone(),
|
||||
tls: vault_config.tls.clone(),
|
||||
},
|
||||
crate::config::BackendConfig::Local(_)
|
||||
| crate::config::BackendConfig::Static(_)
|
||||
| crate::config::BackendConfig::Aws(_) => {
|
||||
crate::config::BackendConfig::Local(_) | crate::config::BackendConfig::Static(_) => {
|
||||
return Err(KmsError::configuration_error("Expected Vault Transit backend configuration"));
|
||||
}
|
||||
};
|
||||
@@ -1280,18 +1235,6 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
self.client.rotate_key(key_id, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.client.update_key_description(key_id, description).await
|
||||
}
|
||||
|
||||
async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
self.client.tag_key(key_id, tags).await
|
||||
}
|
||||
|
||||
async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
self.client.untag_key(key_id, tag_keys).await
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<bool> {
|
||||
self.client.health_check().await.map(|_| true)
|
||||
}
|
||||
@@ -1306,7 +1249,6 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
.with_schedule_deletion(true)
|
||||
.with_versioning(true)
|
||||
.with_physical_delete(true)
|
||||
.with_update_key_metadata(true)
|
||||
}
|
||||
|
||||
async fn remove_expired_key(&self, key_id: &str, now: &Zoned) -> Result<ExpiredKeyRemoval> {
|
||||
|
||||
@@ -74,7 +74,7 @@ pub const LOCAL_BUNDLE_MANIFEST_FILE: &str = "manifest.json";
|
||||
const ARTIFACTS_DIR: &str = "artifacts";
|
||||
const KEYS_DIR: &str = "artifacts/keys";
|
||||
const SALT_ARTIFACT_PATH: &str = "artifacts/master-key.salt.enc";
|
||||
const AEAD_NONCE_LEN: usize = 12;
|
||||
pub(crate) const AEAD_NONCE_LEN: usize = 12;
|
||||
/// Domain-separation context for the artifact AAD binding.
|
||||
const BUNDLE_AAD_CONTEXT: &str = "rustfs-kms-local-backup:v1";
|
||||
/// Domain-separation context for the master-key verifier.
|
||||
@@ -118,7 +118,7 @@ impl BackupKek {
|
||||
}
|
||||
}
|
||||
|
||||
fn cipher(&self) -> Aes256Gcm {
|
||||
pub(crate) fn cipher(&self) -> Aes256Gcm {
|
||||
Aes256Gcm::new(&Key::<Aes256Gcm>::from(*self.key))
|
||||
}
|
||||
}
|
||||
@@ -227,11 +227,14 @@ pub async fn export_local_backup(
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
/// Read and fully validate the manifest of a local bundle directory.
|
||||
/// Read and fully validate the manifest of a bundle directory, whatever
|
||||
/// backend produced it.
|
||||
///
|
||||
/// A directory without a manifest is an interrupted export: the manifest is
|
||||
/// written last, so its absence means the bundle never sealed.
|
||||
pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
|
||||
/// written last, so its absence means the bundle never sealed. The manifest
|
||||
/// file name and framing are bundle-wide, not Local-specific, so consumers of
|
||||
/// other backends' bundles read them through here too.
|
||||
pub async fn read_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
|
||||
let manifest_path = bundle_dir.join(LOCAL_BUNDLE_MANIFEST_FILE);
|
||||
let bytes = match fs::read(&manifest_path).await {
|
||||
Ok(bytes) => bytes,
|
||||
@@ -240,7 +243,12 @@ pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManif
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
};
|
||||
let manifest = BackupManifest::decode(&bytes)?;
|
||||
Ok(BackupManifest::decode(&bytes)?)
|
||||
}
|
||||
|
||||
/// Read and fully validate the manifest of a *local* bundle directory.
|
||||
pub async fn read_local_bundle_manifest(bundle_dir: &Path) -> Result<BackupManifest> {
|
||||
let manifest = read_bundle_manifest(bundle_dir).await?;
|
||||
if manifest.backend != BackupBackendKind::Local {
|
||||
return Err(
|
||||
BackupError::corrupted(format!("bundle manifest declares backend {:?}, expected Local", manifest.backend)).into(),
|
||||
@@ -424,6 +432,7 @@ async fn build_and_write_bundle(
|
||||
backup_kek: kek.descriptor(),
|
||||
artifacts,
|
||||
local_kdf: Some(local_kdf_descriptor(snapshot, master_key_verifier)),
|
||||
external_references: None,
|
||||
key_versions: None,
|
||||
capability_discovery: None,
|
||||
completeness: CompletenessState::InProgress,
|
||||
@@ -502,7 +511,7 @@ async fn encrypt_and_write_artifact(
|
||||
|
||||
/// AAD binding an artifact to its bundle identity and path. A JSON tuple
|
||||
/// gives unambiguous field boundaries without a hand-rolled framing format.
|
||||
fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec<u8> {
|
||||
pub(crate) fn artifact_aad(backup_id: &str, snapshot_generation: u64, artifact_path: &str) -> Vec<u8> {
|
||||
serde_json::to_vec(&(BUNDLE_AAD_CONTEXT, backup_id, snapshot_generation, artifact_path))
|
||||
.expect("AAD tuple of strings and integers always serializes")
|
||||
}
|
||||
|
||||
@@ -306,6 +306,141 @@ impl LocalKdfDescriptor {
|
||||
}
|
||||
}
|
||||
|
||||
/// Transit engine reference recorded in a Vault-backed bundle.
|
||||
///
|
||||
/// Transit keys are non-exportable, so a bundle can only ever name the key and
|
||||
/// the version window its content depends on. Restore compares these values
|
||||
/// against the Transit key the operator's native Vault restore produced.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct VaultTransitReference {
|
||||
/// Transit engine mount path.
|
||||
pub mount_path: String,
|
||||
/// Transit key name.
|
||||
pub key_name: String,
|
||||
/// Lowest Transit key version the bundle's content still needs. A target
|
||||
/// whose `min_decryption_version` sits above this can no longer decrypt
|
||||
/// the oldest state in the bundle.
|
||||
pub required_min_version: u32,
|
||||
/// Latest Transit key version at snapshot time. A target whose newest
|
||||
/// version is below this was restored to a point before the bundle.
|
||||
pub current_version: u32,
|
||||
/// Operator-recorded immutable reference to the native Vault/HSM snapshot
|
||||
/// protecting this key. RustFS neither produces nor consumes that
|
||||
/// snapshot; the reference exists so restore evidence can name it.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub native_snapshot_reference: Option<String>,
|
||||
}
|
||||
|
||||
/// KV generation of one key's record at snapshot time.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct VaultKvRecordReference {
|
||||
/// Stable key id the record belongs to.
|
||||
pub key_id: String,
|
||||
/// KV2 secret version holding the record when the snapshot was taken.
|
||||
/// Restore compares the target's current version against it: a lower
|
||||
/// value means the target's Vault was restored to a point before the
|
||||
/// bundle, a higher value means the target moved ahead of it.
|
||||
pub kv_version: u64,
|
||||
}
|
||||
|
||||
/// Immutable references to the external Vault state a bundle depends on.
|
||||
///
|
||||
/// A Vault-backed bundle never carries the cryptographic root: Transit keys
|
||||
/// cannot be exported and KV2 records live in Vault's own storage. What the
|
||||
/// bundle can own is a precise description of *which* external state it was
|
||||
/// captured against, so a restore refuses to proceed when the operator's
|
||||
/// native Vault restore landed somewhere else. Credentials are structurally
|
||||
/// absent — no token, AppRole secret id, or TLS material is representable in
|
||||
/// this type.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct VaultExternalReferences {
|
||||
/// Opaque identity of the Vault cluster the snapshot was taken from.
|
||||
pub cluster_id: String,
|
||||
/// Vault Enterprise namespace, when the deployment uses one.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub namespace: Option<String>,
|
||||
/// KV v2 mount holding the RustFS records.
|
||||
pub kv_mount: String,
|
||||
/// Path prefix under `kv_mount` holding the RustFS records.
|
||||
pub kv_path_prefix: String,
|
||||
/// Transit engine reference; present exactly when the bundle's material
|
||||
/// is protected by a Transit key.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub transit: Option<VaultTransitReference>,
|
||||
/// Per-key KV generation observed at snapshot time, one entry per key the
|
||||
/// bundle carries a record for.
|
||||
pub kv_records: Vec<VaultKvRecordReference>,
|
||||
}
|
||||
|
||||
impl VaultExternalReferences {
|
||||
fn validate(&self, backend: BackupBackendKind, protection: AtRestProtection) -> Result<(), BackupError> {
|
||||
require_non_empty("external_references.cluster_id", &self.cluster_id)?;
|
||||
require_non_empty("external_references.kv_mount", &self.kv_mount)?;
|
||||
require_non_empty("external_references.kv_path_prefix", &self.kv_path_prefix)?;
|
||||
if self.namespace.as_deref() == Some("") {
|
||||
return Err(BackupError::corrupted("external Vault namespace must not be empty when present"));
|
||||
}
|
||||
|
||||
// The Transit reference is required exactly where the cryptographic
|
||||
// root lives in Transit; a storage-only KV2 bundle that claims one
|
||||
// would misdescribe its own trust root.
|
||||
let transit_required = matches!(
|
||||
(backend, protection),
|
||||
(BackupBackendKind::VaultTransit, _) | (BackupBackendKind::VaultKv2, AtRestProtection::TransitWrapped)
|
||||
);
|
||||
match (&self.transit, transit_required) {
|
||||
(None, true) => {
|
||||
return Err(BackupError::corrupted(format!(
|
||||
"({backend:?}, {protection:?}) bundles must reference the Transit key protecting them"
|
||||
)));
|
||||
}
|
||||
(Some(_), false) => {
|
||||
return Err(BackupError::corrupted(format!(
|
||||
"({backend:?}, {protection:?}) bundles have no Transit trust root to reference"
|
||||
)));
|
||||
}
|
||||
(Some(transit), true) => {
|
||||
require_non_empty("external_references.transit.mount_path", &transit.mount_path)?;
|
||||
require_non_empty("external_references.transit.key_name", &transit.key_name)?;
|
||||
if transit.required_min_version == 0 {
|
||||
return Err(BackupError::corrupted("Transit reference must require at least key version 1"));
|
||||
}
|
||||
if transit.current_version < transit.required_min_version {
|
||||
return Err(BackupError::corrupted(format!(
|
||||
"Transit reference declares current version {} below the required minimum {}",
|
||||
transit.current_version, transit.required_min_version
|
||||
)));
|
||||
}
|
||||
if transit.native_snapshot_reference.as_deref() == Some("") {
|
||||
return Err(BackupError::corrupted("Transit native snapshot reference must not be empty when present"));
|
||||
}
|
||||
}
|
||||
(None, false) => {}
|
||||
}
|
||||
|
||||
let mut seen = BTreeSet::new();
|
||||
for record in &self.kv_records {
|
||||
require_non_empty("external_references.kv_records[].key_id", &record.key_id)?;
|
||||
if record.kv_version == 0 {
|
||||
return Err(BackupError::corrupted(format!(
|
||||
"KV generation for key '{}' must be at least 1",
|
||||
record.key_id
|
||||
)));
|
||||
}
|
||||
if !seen.insert(record.key_id.as_str()) {
|
||||
return Err(BackupError::corrupted(format!(
|
||||
"KV reference for key '{}' appears more than once",
|
||||
record.key_id
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Completeness marker of a bundle.
|
||||
///
|
||||
/// A producer writes `in-progress` state (or no manifest at all) until the
|
||||
@@ -394,6 +529,10 @@ pub struct BackupManifest {
|
||||
/// Local backend section; required exactly when `backend` is `Local`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub local_kdf: Option<LocalKdfDescriptor>,
|
||||
/// External Vault references; required exactly when `backend` is a Vault
|
||||
/// backend. Never carries credentials — see [`VaultExternalReferences`].
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub external_references: Option<VaultExternalReferences>,
|
||||
/// Reserved for the per-key version/envelope inventory defined by the
|
||||
/// backlog#1565 contract. May not carry data in format version 1.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -572,6 +711,7 @@ impl BackupManifest {
|
||||
}
|
||||
self.validate_responsibility()?;
|
||||
self.validate_local_kdf()?;
|
||||
self.validate_external_references()?;
|
||||
self.validate_artifacts()?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -601,6 +741,19 @@ impl BackupManifest {
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_external_references(&self) -> Result<(), BackupError> {
|
||||
let vault_backend = matches!(self.backend, BackupBackendKind::VaultKv2 | BackupBackendKind::VaultTransit);
|
||||
match (&self.external_references, vault_backend) {
|
||||
(None, true) => Err(BackupError::corrupted("Vault backend manifest must carry external Vault references")),
|
||||
(Some(_), false) => Err(BackupError::corrupted(format!(
|
||||
"external Vault references are not valid for backend {:?}",
|
||||
self.backend
|
||||
))),
|
||||
(Some(references), true) => references.validate(self.backend, self.at_rest_protection),
|
||||
(None, false) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_artifacts(&self) -> Result<(), BackupError> {
|
||||
let mut paths = BTreeSet::new();
|
||||
for artifact in &self.artifacts {
|
||||
@@ -865,6 +1018,7 @@ mod tests {
|
||||
vec![AtRestProtection::EncryptedMasterKey],
|
||||
Some("verifier-opaque-1".to_string()),
|
||||
)),
|
||||
external_references: None,
|
||||
key_versions: None,
|
||||
capability_discovery: None,
|
||||
completeness: CompletenessState::InProgress,
|
||||
@@ -1271,4 +1425,161 @@ mod tests {
|
||||
assert_eq!(decoded, sealed);
|
||||
assert_eq!(decoded.responsibility, BackupResponsibility::ReferenceOnly);
|
||||
}
|
||||
|
||||
fn transit_references() -> VaultExternalReferences {
|
||||
VaultExternalReferences {
|
||||
cluster_id: "vault-cluster-a".to_string(),
|
||||
namespace: Some("tenant-a".to_string()),
|
||||
kv_mount: "secret".to_string(),
|
||||
kv_path_prefix: "rustfs/kms/transit-metadata".to_string(),
|
||||
transit: Some(VaultTransitReference {
|
||||
mount_path: "transit".to_string(),
|
||||
key_name: "rustfs-master".to_string(),
|
||||
required_min_version: 2,
|
||||
current_version: 5,
|
||||
native_snapshot_reference: Some("s3://dr/vault/2026-08-01.snap".to_string()),
|
||||
}),
|
||||
kv_records: vec![VaultKvRecordReference {
|
||||
key_id: "object-key".to_string(),
|
||||
kv_version: 4,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn transit_manifest_unsealed() -> BackupManifest {
|
||||
BackupManifest {
|
||||
backend: BackupBackendKind::VaultTransit,
|
||||
at_rest_protection: AtRestProtection::ExternalNonExportable,
|
||||
responsibility: BackupResponsibility::MetadataPlusExternalRoot,
|
||||
artifacts: vec![artifact(ArtifactKind::KeyMetadata, "vault/records/object-key.json.enc")],
|
||||
local_kdf: None,
|
||||
external_references: Some(transit_references()),
|
||||
..local_manifest_unsealed()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vault_bundle_round_trips_with_external_references() {
|
||||
let sealed = seal(transit_manifest_unsealed());
|
||||
let bytes = sealed.encode().expect("encoding should succeed");
|
||||
let decoded = BackupManifest::decode(&bytes).expect("decoding should succeed");
|
||||
assert_eq!(decoded, sealed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vault_backend_requires_external_references() {
|
||||
let manifest = BackupManifest {
|
||||
external_references: None,
|
||||
..transit_manifest_unsealed()
|
||||
};
|
||||
expect_corrupted(
|
||||
BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")),
|
||||
"must carry external Vault references",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_vault_backend_rejects_external_references() {
|
||||
let manifest = BackupManifest {
|
||||
external_references: Some(transit_references()),
|
||||
..local_manifest_unsealed()
|
||||
};
|
||||
expect_corrupted(
|
||||
BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")),
|
||||
"not valid for backend",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_reference_contradictions_fail_closed() {
|
||||
let cases: [(&str, Box<dyn Fn(&mut VaultExternalReferences)>); 7] = [
|
||||
("cluster_id", Box::new(|refs| refs.cluster_id.clear())),
|
||||
("kv_mount", Box::new(|refs| refs.kv_mount.clear())),
|
||||
("namespace must not be empty", Box::new(|refs| refs.namespace = Some(String::new()))),
|
||||
(
|
||||
"must reference the Transit key",
|
||||
Box::new(|refs: &mut VaultExternalReferences| refs.transit = None),
|
||||
),
|
||||
(
|
||||
"at least key version 1",
|
||||
Box::new(|refs: &mut VaultExternalReferences| {
|
||||
if let Some(transit) = refs.transit.as_mut() {
|
||||
transit.required_min_version = 0;
|
||||
}
|
||||
}),
|
||||
),
|
||||
(
|
||||
"below the required minimum",
|
||||
Box::new(|refs: &mut VaultExternalReferences| {
|
||||
if let Some(transit) = refs.transit.as_mut() {
|
||||
transit.current_version = 1;
|
||||
}
|
||||
}),
|
||||
),
|
||||
(
|
||||
"appears more than once",
|
||||
Box::new(|refs: &mut VaultExternalReferences| {
|
||||
let first = refs.kv_records[0].clone();
|
||||
refs.kv_records.push(first);
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (needle, mutate) in cases {
|
||||
let mut references = transit_references();
|
||||
mutate(&mut references);
|
||||
let manifest = BackupManifest {
|
||||
external_references: Some(references),
|
||||
..transit_manifest_unsealed()
|
||||
};
|
||||
expect_corrupted(BackupManifest::decode(&serde_json::to_vec(&seal(manifest)).expect("serialize")), needle);
|
||||
}
|
||||
}
|
||||
|
||||
/// The reference schema is the only place a Vault coordinate reaches a
|
||||
/// bundle, so its field set is pinned: a credential-carrying field could
|
||||
/// only appear by editing this assertion.
|
||||
#[test]
|
||||
fn external_reference_field_set_is_frozen() {
|
||||
let value = serde_json::to_value(transit_references()).expect("serialize");
|
||||
let mut top: Vec<&str> = value.as_object().expect("object").keys().map(String::as_str).collect();
|
||||
top.sort_unstable();
|
||||
assert_eq!(
|
||||
top,
|
||||
[
|
||||
"cluster_id",
|
||||
"kv_mount",
|
||||
"kv_path_prefix",
|
||||
"kv_records",
|
||||
"namespace",
|
||||
"transit"
|
||||
]
|
||||
);
|
||||
|
||||
let mut transit: Vec<&str> = value["transit"]
|
||||
.as_object()
|
||||
.expect("transit object")
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect();
|
||||
transit.sort_unstable();
|
||||
assert_eq!(
|
||||
transit,
|
||||
[
|
||||
"current_version",
|
||||
"key_name",
|
||||
"mount_path",
|
||||
"native_snapshot_reference",
|
||||
"required_min_version"
|
||||
]
|
||||
);
|
||||
|
||||
let mut record: Vec<&str> = value["kv_records"][0]
|
||||
.as_object()
|
||||
.expect("record object")
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect();
|
||||
record.sort_unstable();
|
||||
assert_eq!(record, ["key_id", "kv_version"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,9 +17,11 @@
|
||||
//! The contract side defines the versioned backup manifest, the per-backend
|
||||
//! responsibility matrix, typed failure modes, and the restore dry-run
|
||||
//! report. [`local_export`] implements the producer side and
|
||||
//! [`local_restore`] the consumer side for the Local backend as
|
||||
//! crate-internal APIs; the admin API builds on these pieces in follow-up
|
||||
//! changes.
|
||||
//! [`local_restore`] the consumer side for the Local backend;
|
||||
//! [`vault_restore`] orchestrates the consumer side for the Vault backends,
|
||||
//! whose cryptographic root is restored by Vault's own disaster-recovery
|
||||
//! flow. All are crate-internal APIs; the admin API builds on these pieces in
|
||||
//! follow-up changes.
|
||||
//!
|
||||
//! # Bundle model
|
||||
//!
|
||||
@@ -54,6 +56,7 @@ mod error;
|
||||
pub mod local_export;
|
||||
pub mod local_restore;
|
||||
mod manifest;
|
||||
pub mod vault_restore;
|
||||
|
||||
pub use capability::{AtRestProtection, BackupBackendKind, BackupResponsibility};
|
||||
pub use dry_run::{
|
||||
@@ -62,7 +65,7 @@ pub use dry_run::{
|
||||
pub use error::BackupError;
|
||||
pub use local_export::{
|
||||
BackupKek, LOCAL_BUNDLE_MANIFEST_FILE, LocalBackupExportRequest, decrypt_bundle_artifact, export_local_backup,
|
||||
read_local_bundle_manifest,
|
||||
read_bundle_manifest, read_local_bundle_manifest,
|
||||
};
|
||||
pub use local_restore::{
|
||||
LocalRestoreReport, LocalRestoreRequest, RestoreConflictPolicy, abort_local_restore, dry_run_local_restore,
|
||||
@@ -70,5 +73,10 @@ pub use local_restore::{
|
||||
};
|
||||
pub use manifest::{
|
||||
AeadAlgorithm, ArtifactDescriptor, ArtifactKind, BackupKekDescriptor, BackupManifest, CompletenessState, ContentDigest,
|
||||
DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot,
|
||||
DigestAlgorithm, LocalKdfDescriptor, LocalKeyDerivation, ReservedSlot, VaultExternalReferences, VaultKvRecordReference,
|
||||
VaultTransitReference,
|
||||
};
|
||||
pub use vault_restore::{
|
||||
VaultRestoreClient, VaultRestoreMismatch, VaultRestoreReport, VaultRestoreRequest, VaultRestoreSequence, VaultRestoreStage,
|
||||
VaultRestoreTarget, abort_vault_restore, dry_run_vault_restore, restore_vault_backup,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+38
-306
@@ -16,84 +16,11 @@
|
||||
|
||||
use crate::types::KeyMetadata;
|
||||
use moka::future::Cache;
|
||||
use moka::notification::RemovalCause;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Default lifetime of a cached key metadata entry.
|
||||
const DEFAULT_METADATA_TTL: Duration = Duration::from_secs(300);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metrics
|
||||
//
|
||||
// Lookup outcomes and removals are counted here because moka exposes neither
|
||||
// hit nor miss counts; the numbers below are the cache's own, not derived.
|
||||
// Label values are exclusively static strings (lookup result, removal cause) —
|
||||
// key identifiers and key metadata must never reach a metric label.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Counter: key metadata lookups, by `result` (`hit` or `miss`).
|
||||
const METRIC_CACHE_LOOKUPS_TOTAL: &str = "rustfs_kms_metadata_cache_lookups_total";
|
||||
/// Counter: entries dropped from the cache, by `cause` (`expired`, `size`,
|
||||
/// `explicit`, `replaced`); only `expired` and `size` are true evictions, the
|
||||
/// rest are invalidations driven by key lifecycle operations.
|
||||
const METRIC_CACHE_EVICTIONS_TOTAL: &str = "rustfs_kms_metadata_cache_evictions_total";
|
||||
/// Gauge: entries currently held by the cache. Republished whenever the entry
|
||||
/// set can have changed — every write path, plus the lookups that miss, since
|
||||
/// TTL expiry drops entries without any write taking place.
|
||||
const METRIC_CACHE_ENTRIES: &str = "rustfs_kms_metadata_cache_entries";
|
||||
|
||||
/// Register metric descriptions once per process.
|
||||
fn describe_metrics() {
|
||||
static DESCRIBE: std::sync::Once = std::sync::Once::new();
|
||||
DESCRIBE.call_once(|| {
|
||||
metrics::describe_counter!(METRIC_CACHE_LOOKUPS_TOTAL, "Total KMS key metadata cache lookups, by hit or miss");
|
||||
metrics::describe_counter!(
|
||||
METRIC_CACHE_EVICTIONS_TOTAL,
|
||||
"Total entries dropped from the KMS key metadata cache, by removal cause"
|
||||
);
|
||||
metrics::describe_gauge!(METRIC_CACHE_ENTRIES, "Entries currently held by the KMS key metadata cache");
|
||||
});
|
||||
}
|
||||
|
||||
fn removal_cause_label(cause: RemovalCause) -> &'static str {
|
||||
match cause {
|
||||
RemovalCause::Expired => "expired",
|
||||
RemovalCause::Explicit => "explicit",
|
||||
RemovalCause::Replaced => "replaced",
|
||||
RemovalCause::Size => "size",
|
||||
}
|
||||
}
|
||||
|
||||
/// Cumulative cache counters. Shared with moka's eviction listener, which runs
|
||||
/// outside any `&self` borrow, hence the `Arc` and the atomics.
|
||||
#[derive(Debug, Default)]
|
||||
struct CacheCounters {
|
||||
hits: AtomicU64,
|
||||
misses: AtomicU64,
|
||||
evictions: AtomicU64,
|
||||
}
|
||||
|
||||
/// Snapshot of the KMS key metadata cache counters.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct KmsCacheStats {
|
||||
/// Entries currently held. Eventually consistent: moka applies pending
|
||||
/// maintenance lazily.
|
||||
pub entries: u64,
|
||||
/// Lookups served from the cache since process start.
|
||||
pub hits: u64,
|
||||
/// Lookups that fell through to the backend since process start.
|
||||
pub misses: u64,
|
||||
/// Entries dropped since process start, whatever the cause (expiry,
|
||||
/// capacity, invalidation, replacement).
|
||||
pub evictions: u64,
|
||||
}
|
||||
|
||||
/// KMS cache for storing frequently accessed keys and metadata
|
||||
pub struct KmsCache {
|
||||
key_metadata_cache: Cache<String, KeyMetadata>,
|
||||
counters: Arc<CacheCounters>,
|
||||
}
|
||||
|
||||
impl KmsCache {
|
||||
@@ -106,26 +33,11 @@ impl KmsCache {
|
||||
/// A new instance of `KmsCache`
|
||||
///
|
||||
pub fn new(capacity: u64) -> Self {
|
||||
Self::with_ttl(capacity, DEFAULT_METADATA_TTL)
|
||||
}
|
||||
|
||||
/// Create a new KMS cache with an explicit metadata time-to-live
|
||||
fn with_ttl(capacity: u64, metadata_ttl: Duration) -> Self {
|
||||
describe_metrics();
|
||||
|
||||
let counters = Arc::new(CacheCounters::default());
|
||||
let eviction_counters = Arc::clone(&counters);
|
||||
|
||||
Self {
|
||||
key_metadata_cache: Cache::builder()
|
||||
.max_capacity(capacity)
|
||||
.time_to_live(metadata_ttl)
|
||||
.eviction_listener(move |_key: Arc<String>, _metadata: KeyMetadata, cause: RemovalCause| {
|
||||
eviction_counters.evictions.fetch_add(1, Ordering::Relaxed);
|
||||
metrics::counter!(METRIC_CACHE_EVICTIONS_TOTAL, "cause" => removal_cause_label(cause)).increment(1);
|
||||
})
|
||||
.time_to_live(Duration::from_secs(300)) // 5 minutes default TTL
|
||||
.build(),
|
||||
counters,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,26 +50,7 @@ impl KmsCache {
|
||||
/// An `Option` containing the `KeyMetadata` if found, or `None` if not found
|
||||
///
|
||||
pub async fn get_key_metadata(&self, key_id: &str) -> Option<KeyMetadata> {
|
||||
let cached = self.key_metadata_cache.get(key_id).await;
|
||||
self.record_lookup(cached.is_some());
|
||||
|
||||
if cached.is_none() {
|
||||
// TTL expiry is the one way the entry set shrinks without a write,
|
||||
// and a miss is where it surfaces. moka expires lazily: the miss is
|
||||
// reported at once, but the removal that decrements `entry_count`
|
||||
// and reaches the eviction listener lands in the maintenance pass
|
||||
// moka runs opportunistically on reads, on its own interval. So
|
||||
// this converges the gauge within a housekeeping interval rather
|
||||
// than on the first miss — enough to stop a cache that only ever
|
||||
// expires, with no put, remove or clear to follow, from reporting a
|
||||
// population that is gone. Forcing `run_pending_tasks` would
|
||||
// tighten that at the cost of taking moka's maintenance lock on
|
||||
// every miss, for a gauge `entry_count` only approximates anyway.
|
||||
// Hits, the hot path, are left alone.
|
||||
self.record_entry_count();
|
||||
}
|
||||
|
||||
cached
|
||||
self.key_metadata_cache.get(key_id).await
|
||||
}
|
||||
|
||||
/// Put key metadata into cache
|
||||
@@ -169,7 +62,6 @@ impl KmsCache {
|
||||
pub async fn put_key_metadata(&mut self, key_id: &str, metadata: &KeyMetadata) {
|
||||
self.key_metadata_cache.insert(key_id.to_string(), metadata.clone()).await;
|
||||
self.key_metadata_cache.run_pending_tasks().await;
|
||||
self.record_entry_count();
|
||||
}
|
||||
|
||||
/// Remove key metadata from cache
|
||||
@@ -179,11 +71,6 @@ impl KmsCache {
|
||||
///
|
||||
pub async fn remove_key_metadata(&mut self, key_id: &str) {
|
||||
self.key_metadata_cache.remove(key_id).await;
|
||||
|
||||
// Flush maintenance so the removal notification and the entry gauge
|
||||
// describe the cache as it is once this call returns.
|
||||
self.key_metadata_cache.run_pending_tasks().await;
|
||||
self.record_entry_count();
|
||||
}
|
||||
|
||||
/// Clear all cached entries
|
||||
@@ -192,35 +79,18 @@ impl KmsCache {
|
||||
|
||||
// Wait for invalidation to complete
|
||||
self.key_metadata_cache.run_pending_tasks().await;
|
||||
self.record_entry_count();
|
||||
}
|
||||
|
||||
/// Get cache statistics
|
||||
/// Get cache statistics (hit count, miss count)
|
||||
///
|
||||
/// # Returns
|
||||
/// A [`KmsCacheStats`] snapshot of entry count, hits, misses and evictions
|
||||
/// A tuple containing total entries and total misses
|
||||
///
|
||||
pub fn stats(&self) -> KmsCacheStats {
|
||||
KmsCacheStats {
|
||||
entries: self.key_metadata_cache.entry_count(),
|
||||
hits: self.counters.hits.load(Ordering::Relaxed),
|
||||
misses: self.counters.misses.load(Ordering::Relaxed),
|
||||
evictions: self.counters.evictions.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
fn record_lookup(&self, hit: bool) {
|
||||
let (counter, result) = if hit {
|
||||
(&self.counters.hits, "hit")
|
||||
} else {
|
||||
(&self.counters.misses, "miss")
|
||||
};
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
metrics::counter!(METRIC_CACHE_LOOKUPS_TOTAL, "result" => result).increment(1);
|
||||
}
|
||||
|
||||
fn record_entry_count(&self) {
|
||||
metrics::gauge!(METRIC_CACHE_ENTRIES).set(self.key_metadata_cache.entry_count() as f64);
|
||||
pub fn stats(&self) -> (u64, u64) {
|
||||
(
|
||||
self.key_metadata_cache.entry_count(),
|
||||
0u64, // moka doesn't provide miss count directly
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,90 +99,37 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::types::{KeyState, KeyUsage};
|
||||
use jiff::Zoned;
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct CacheInfo {
|
||||
key_metadata_count: u64,
|
||||
}
|
||||
|
||||
impl CacheInfo {
|
||||
fn total_entries(&self) -> u64 {
|
||||
self.key_metadata_count
|
||||
}
|
||||
}
|
||||
|
||||
impl KmsCache {
|
||||
fn with_ttl_for_tests(capacity: u64, metadata_ttl: Duration) -> Self {
|
||||
Self {
|
||||
key_metadata_cache: Cache::builder().max_capacity(capacity).time_to_live(metadata_ttl).build(),
|
||||
}
|
||||
}
|
||||
|
||||
fn info_for_tests(&self) -> CacheInfo {
|
||||
CacheInfo {
|
||||
key_metadata_count: self.key_metadata_cache.entry_count(),
|
||||
}
|
||||
}
|
||||
|
||||
fn contains_key_metadata_for_tests(&self, key_id: &str) -> bool {
|
||||
self.key_metadata_cache.contains_key(key_id)
|
||||
}
|
||||
}
|
||||
|
||||
fn test_metadata(key_id: &str) -> KeyMetadata {
|
||||
KeyMetadata {
|
||||
key_id: key_id.to_string(),
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "KMS".to_string(),
|
||||
key_manager: "CUSTOMER".to_string(),
|
||||
tags: std::collections::HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
type MetricEntry = (
|
||||
metrics_util::CompositeKey,
|
||||
Option<metrics::Unit>,
|
||||
Option<metrics::SharedString>,
|
||||
DebugValue,
|
||||
);
|
||||
|
||||
/// Drive `test` on a current-thread runtime under a thread-local debugging
|
||||
/// recorder and return its output plus one snapshot of everything emitted.
|
||||
///
|
||||
/// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
|
||||
/// the recorded state, so taking it per assertion would leave every
|
||||
/// assertion after the first with nothing to look at.
|
||||
fn record_metrics<F, Fut, T>(test: F) -> (T, Vec<MetricEntry>)
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = T>,
|
||||
{
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let output = metrics::with_local_recorder(&recorder, || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("current-thread runtime must build");
|
||||
runtime.block_on(test())
|
||||
});
|
||||
(output, snapshotter.snapshot().into_vec())
|
||||
}
|
||||
|
||||
/// Sum of counters with `name` whose labels include every `(key, value)` pair.
|
||||
fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
|
||||
snapshot
|
||||
.iter()
|
||||
.filter_map(|(composite, _unit, _description, value)| {
|
||||
let key = composite.key();
|
||||
let matches = composite.kind() == MetricKind::Counter
|
||||
&& key.name() == name
|
||||
&& labels
|
||||
.iter()
|
||||
.all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected));
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Counter(count)) => Some(*count),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Last value recorded for the unlabelled gauge `name`.
|
||||
fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
|
||||
snapshot.iter().find_map(|(composite, _unit, _description, value)| {
|
||||
let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Gauge(gauge)) => Some(**gauge),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cache_operations() {
|
||||
let mut cache = KmsCache::new(100);
|
||||
@@ -337,16 +154,19 @@ mod tests {
|
||||
assert_eq!(retrieved.expect("metadata should be cached").key_id, "test-key-1");
|
||||
|
||||
// Test cache info
|
||||
assert_eq!(cache.stats().entries, 1);
|
||||
let info = cache.info_for_tests();
|
||||
assert_eq!(info.key_metadata_count, 1);
|
||||
assert_eq!(info.total_entries(), 1);
|
||||
|
||||
// Test cache clearing
|
||||
cache.clear().await;
|
||||
assert_eq!(cache.stats().entries, 0);
|
||||
let info_after_clear = cache.info_for_tests();
|
||||
assert_eq!(info_after_clear.total_entries(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cache_with_custom_ttl() {
|
||||
let mut cache = KmsCache::with_ttl(
|
||||
let mut cache = KmsCache::with_ttl_for_tests(
|
||||
100,
|
||||
Duration::from_millis(100), // Short TTL for testing
|
||||
);
|
||||
@@ -397,92 +217,4 @@ mod tests {
|
||||
|
||||
assert!(cache.contains_key_metadata_for_tests("contains-test"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lookups_report_real_hit_and_miss_counts() {
|
||||
let (stats, snapshot) = record_metrics(|| async {
|
||||
let mut cache = KmsCache::new(100);
|
||||
|
||||
assert!(cache.get_key_metadata("absent").await.is_none());
|
||||
cache.put_key_metadata("present", &test_metadata("present")).await;
|
||||
assert!(cache.get_key_metadata("present").await.is_some());
|
||||
assert!(cache.get_key_metadata("absent").await.is_none());
|
||||
|
||||
cache.stats()
|
||||
});
|
||||
|
||||
assert_eq!(stats.hits, 1);
|
||||
assert_eq!(stats.misses, 2);
|
||||
assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "hit")]), 1);
|
||||
assert_eq!(counter_value(&snapshot, METRIC_CACHE_LOOKUPS_TOTAL, &[("result", "miss")]), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn removals_report_their_cause_and_the_resulting_entry_count() {
|
||||
let (stats, snapshot) = record_metrics(|| async {
|
||||
let mut cache = KmsCache::new(100);
|
||||
|
||||
cache.put_key_metadata("key", &test_metadata("key")).await;
|
||||
cache.put_key_metadata("key", &test_metadata("key")).await;
|
||||
cache.remove_key_metadata("key").await;
|
||||
|
||||
cache.stats()
|
||||
});
|
||||
|
||||
assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "replaced")]), 1);
|
||||
assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "explicit")]), 1);
|
||||
assert_eq!(stats.evictions, 2);
|
||||
assert_eq!(stats.entries, 0);
|
||||
assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capacity_pressure_reports_size_evictions() {
|
||||
let (stats, snapshot) = record_metrics(|| async {
|
||||
let mut cache = KmsCache::with_ttl(1, DEFAULT_METADATA_TTL);
|
||||
|
||||
cache.put_key_metadata("first", &test_metadata("first")).await;
|
||||
cache.put_key_metadata("second", &test_metadata("second")).await;
|
||||
|
||||
cache.stats()
|
||||
});
|
||||
|
||||
assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "size")]), 1);
|
||||
assert_eq!(stats.evictions, 1);
|
||||
assert_eq!(stats.entries, 1);
|
||||
assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(1.0));
|
||||
}
|
||||
|
||||
/// Entries that expire are dropped outside every write path, so the gauge
|
||||
/// has to be republished from the lookup that observes the expiry — nothing
|
||||
/// else runs afterwards to correct it.
|
||||
///
|
||||
/// moka's clock is internal and cannot be driven by tokio's paused time,
|
||||
/// hence the one short real sleep. The explicit `run_pending_tasks` stands
|
||||
/// in for the maintenance moka schedules by itself on reads, so the test
|
||||
/// does not depend on moka's internal 300ms housekeeping interval.
|
||||
#[test]
|
||||
fn expiry_without_further_writes_converges_the_entry_gauge() {
|
||||
let ttl = Duration::from_millis(100);
|
||||
|
||||
let (stats, snapshot) = record_metrics(move || async move {
|
||||
let mut cache = KmsCache::with_ttl(100, ttl);
|
||||
|
||||
cache.put_key_metadata("expiring", &test_metadata("expiring")).await;
|
||||
assert_eq!(cache.stats().entries, 1);
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
cache.key_metadata_cache.run_pending_tasks().await;
|
||||
|
||||
// The lookup that observes the expiry is the last thing to touch
|
||||
// the cache: no put, remove or clear follows it.
|
||||
assert!(cache.get_key_metadata("expiring").await.is_none());
|
||||
|
||||
cache.stats()
|
||||
});
|
||||
|
||||
assert_eq!(counter_value(&snapshot, METRIC_CACHE_EVICTIONS_TOTAL, &[("cause", "expired")]), 1);
|
||||
assert_eq!(stats.entries, 0);
|
||||
assert_eq!(gauge_value(&snapshot, METRIC_CACHE_ENTRIES), Some(0.0));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,8 +34,6 @@ pub const ENV_KMS_VAULT_APPROLE_SECRET_ID: &str = "RUSTFS_KMS_VAULT_APPROLE_SECR
|
||||
pub const ENV_KMS_VAULT_APPROLE_SECRET_ID_FILE: &str = "RUSTFS_KMS_VAULT_APPROLE_SECRET_ID_FILE";
|
||||
pub const ENV_KMS_VAULT_APPROLE_MOUNT: &str = "RUSTFS_KMS_VAULT_APPROLE_MOUNT";
|
||||
pub const ENV_KMS_VAULT_TOKEN_FILE: &str = "RUSTFS_KMS_VAULT_TOKEN_FILE";
|
||||
pub const ENV_KMS_AWS_REGION: &str = "RUSTFS_KMS_AWS_REGION";
|
||||
pub const ENV_KMS_AWS_ENDPOINT_URL: &str = "RUSTFS_KMS_AWS_ENDPOINT_URL";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KV_MOUNT: &str = "secret";
|
||||
pub const DEFAULT_VAULT_TRANSIT_METADATA_KEY_PREFIX: &str = "rustfs/kms/transit-metadata";
|
||||
pub const DEFAULT_VAULT_APPROLE_MOUNT: &str = "approle";
|
||||
@@ -130,26 +128,6 @@ pub enum KmsBackend {
|
||||
/// Static single-key backend that derives DEKs from a pre-configured key
|
||||
#[serde(rename = "Static")]
|
||||
Static,
|
||||
/// AWS KMS backend: AWS is the cryptographic source of truth and owns key
|
||||
/// state, versioning, and the deletion window.
|
||||
#[serde(rename = "AWS", alias = "AwsKms")]
|
||||
Aws,
|
||||
}
|
||||
|
||||
impl KmsBackend {
|
||||
/// Stable identifier for logs, metrics, and audit records.
|
||||
///
|
||||
/// External consumers key off these values, so treat them as a wire
|
||||
/// contract rather than a rendering detail.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
KmsBackend::VaultKv2 => "vault-kv2",
|
||||
KmsBackend::VaultTransit => "vault-transit",
|
||||
KmsBackend::Local => "local",
|
||||
KmsBackend::Static => "static",
|
||||
KmsBackend::Aws => "aws",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Main KMS configuration
|
||||
@@ -207,9 +185,6 @@ pub enum BackendConfig {
|
||||
VaultTransit(Box<VaultTransitConfig>),
|
||||
/// Static single-key backend configuration
|
||||
Static(StaticConfig),
|
||||
/// AWS KMS backend configuration
|
||||
#[serde(rename = "AWS", alias = "AwsKms")]
|
||||
Aws(Box<AwsKmsConfig>),
|
||||
}
|
||||
|
||||
impl Default for BackendConfig {
|
||||
@@ -225,7 +200,6 @@ impl fmt::Debug for BackendConfig {
|
||||
Self::VaultKv2(config) => f.debug_tuple("VaultKv2").field(config).finish(),
|
||||
Self::VaultTransit(config) => f.debug_tuple("VaultTransit").field(config).finish(),
|
||||
Self::Static(config) => f.debug_tuple("Static").field(config).finish(),
|
||||
Self::Aws(config) => f.debug_tuple("Aws").field(config).finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -551,25 +525,6 @@ impl Default for CacheConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// AWS KMS backend configuration.
|
||||
///
|
||||
/// Deliberately holds no credential material: the backend resolves credentials
|
||||
/// through the standard `aws-config` provider chain (environment, shared
|
||||
/// profile, container/IMDS role), so RustFS never stores, persists, or redacts
|
||||
/// AWS secrets of its own.
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct AwsKmsConfig {
|
||||
/// AWS region hosting the KMS keys. When unset, the region is resolved by
|
||||
/// the standard chain (`AWS_REGION`, profile, IMDS).
|
||||
#[serde(default)]
|
||||
pub region: Option<String>,
|
||||
/// Override for the KMS endpoint, for local emulators and private
|
||||
/// endpoints. Unset in production, where the SDK derives the regional
|
||||
/// endpoint.
|
||||
#[serde(default)]
|
||||
pub endpoint_url: Option<String>,
|
||||
}
|
||||
|
||||
impl KmsConfig {
|
||||
/// Create a new KMS configuration for local backend (for development and testing only)
|
||||
pub fn local(key_dir: PathBuf) -> Self {
|
||||
@@ -679,29 +634,6 @@ impl KmsConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new KMS configuration for the AWS KMS backend.
|
||||
///
|
||||
/// Credentials are resolved by the standard `aws-config` provider chain;
|
||||
/// only the region is configured here.
|
||||
pub fn aws(region: Option<String>) -> Self {
|
||||
Self {
|
||||
backend: KmsBackend::Aws,
|
||||
backend_config: BackendConfig::Aws(Box::new(AwsKmsConfig {
|
||||
region,
|
||||
endpoint_url: None,
|
||||
})),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the AWS configuration if backend is AWS KMS
|
||||
pub fn aws_kms_config(&self) -> Option<&AwsKmsConfig> {
|
||||
match &self.backend_config {
|
||||
BackendConfig::Aws(config) => Some(config),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set default key ID
|
||||
pub fn with_default_key(mut self, key_id: String) -> Self {
|
||||
self.default_key_id = Some(key_id);
|
||||
@@ -858,25 +790,6 @@ impl KmsConfig {
|
||||
// Validate that the key can be decoded (right length, valid base64)
|
||||
config.decode_key()?;
|
||||
}
|
||||
BackendConfig::Aws(config) => {
|
||||
if let Some(region) = &config.region
|
||||
&& region.is_empty()
|
||||
{
|
||||
return Err(KmsError::configuration_error("AWS KMS region cannot be empty when set"));
|
||||
}
|
||||
|
||||
if let Some(endpoint) = &config.endpoint_url {
|
||||
if !endpoint.starts_with("http://") && !endpoint.starts_with("https://") {
|
||||
return Err(KmsError::configuration_error("AWS KMS endpoint URL must use http or https scheme"));
|
||||
}
|
||||
// A plaintext endpoint override exposes every KMS request,
|
||||
// including plaintext data keys, so it stays gated on the
|
||||
// explicit development opt-in.
|
||||
if endpoint.starts_with("http://") && !self.allow_insecure_dev_defaults {
|
||||
return Err(development_default_error("AWS KMS endpoint URL must use https"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate cache configuration
|
||||
@@ -898,7 +811,6 @@ impl KmsConfig {
|
||||
"vault" | "vault-kv2" | "vault_kv2" => KmsBackend::VaultKv2,
|
||||
"vault-transit" | "vault_transit" => KmsBackend::VaultTransit,
|
||||
"static" => KmsBackend::Static,
|
||||
"aws" | "aws-kms" | "aws_kms" => KmsBackend::Aws,
|
||||
_ => return Err(KmsError::configuration_error(format!("Unknown KMS backend: {backend_type}"))),
|
||||
};
|
||||
}
|
||||
@@ -1027,14 +939,6 @@ impl KmsConfig {
|
||||
});
|
||||
config.default_key_id = Some(key_id);
|
||||
}
|
||||
KmsBackend::Aws => {
|
||||
// Only non-credential settings are read here; access keys,
|
||||
// profiles, and role assumption stay with the aws-config chain.
|
||||
config.backend_config = BackendConfig::Aws(Box::new(AwsKmsConfig {
|
||||
region: get_env_opt_str(ENV_KMS_AWS_REGION),
|
||||
endpoint_url: get_env_opt_str(ENV_KMS_AWS_ENDPOINT_URL),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
config.validate()?;
|
||||
@@ -1636,43 +1540,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The AWS backend reads only non-credential settings from the
|
||||
/// environment; access keys, profiles, and role assumption stay with the
|
||||
/// aws-config provider chain.
|
||||
#[test]
|
||||
fn test_from_env_selects_aws_backend_without_credentials() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("aws")),
|
||||
(ENV_KMS_AWS_REGION, Some("eu-central-1")),
|
||||
(ENV_KMS_AWS_ENDPOINT_URL, None::<&str>),
|
||||
],
|
||||
|| {
|
||||
let config = KmsConfig::from_env().expect("kms config should load from env");
|
||||
assert_eq!(config.backend, KmsBackend::Aws);
|
||||
let aws = config.aws_kms_config().expect("aws backend config");
|
||||
assert_eq!(aws.region.as_deref(), Some("eu-central-1"));
|
||||
assert_eq!(aws.endpoint_url, None);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/// A plaintext endpoint override exposes every KMS request, including the
|
||||
/// plaintext data keys, so it stays behind the explicit development opt-in.
|
||||
#[test]
|
||||
fn test_from_env_rejects_plaintext_aws_endpoint() {
|
||||
with_vars(
|
||||
vec![
|
||||
("RUSTFS_KMS_BACKEND", Some("aws")),
|
||||
(ENV_KMS_AWS_REGION, Some("us-east-1")),
|
||||
(ENV_KMS_AWS_ENDPOINT_URL, Some("http://localhost:4566")),
|
||||
],
|
||||
|| {
|
||||
KmsConfig::from_env().expect_err("a plaintext AWS endpoint must be rejected by default");
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_env_approle_requires_secret_id_or_file() {
|
||||
with_vars(
|
||||
|
||||
@@ -22,123 +22,18 @@
|
||||
//! every node of a deployment concurrently — a key is only ever removed while
|
||||
//! its (re-read) record is an expired pending deletion or a tombstone.
|
||||
|
||||
use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
|
||||
use crate::backends::{ExpiredKeyRemoval, KmsBackend};
|
||||
use crate::error::Result;
|
||||
use crate::types::{KeyInfo, KeyStatus, ListKeysRequest, OperationContext};
|
||||
use crate::types::{KeyStatus, ListKeysRequest};
|
||||
use async_trait::async_trait;
|
||||
use jiff::Zoned;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::time::Duration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// How often the worker looks for expired pending deletions.
|
||||
pub const DEFAULT_SWEEP_INTERVAL: Duration = Duration::from_secs(60);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metrics
|
||||
//
|
||||
// The sweep already pages through the whole key set, so everything below is
|
||||
// derived from the pages it has in hand: observing the key lifecycle costs no
|
||||
// extra backend call. Each metric is an aggregate — the gauges carry no labels
|
||||
// at all and the counter only a static outcome — because a per-key label would
|
||||
// carry key identifiers into the metric stream and grow the series count with
|
||||
// the key set. "This key is overdue for rotation" is a threshold on the
|
||||
// aggregate age and belongs in an alerting rule, not in a label.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Gauge: keys scheduled for deletion whose deadline has not passed, as of the
|
||||
/// end of the last sweep that saw the whole key set.
|
||||
const METRIC_PENDING_DELETION_KEYS: &str = "rustfs_kms_pending_deletion_keys";
|
||||
/// Gauge: keys left tombstoned by an interrupted removal and still awaiting
|
||||
/// the sweep, as of the end of the last sweep that saw the whole key set.
|
||||
const METRIC_TOMBSTONE_KEYS: &str = "rustfs_kms_deletion_tombstone_keys";
|
||||
/// Gauge: seconds since the least recently rotated usable key was rotated
|
||||
/// (its creation time when it was never rotated); `0` when there are none.
|
||||
const METRIC_OLDEST_ROTATION_AGE_SECONDS: &str = "rustfs_kms_oldest_key_rotation_age_seconds";
|
||||
/// Counter: keys the sweep acted on, by `outcome` (`removed`, `blocked`,
|
||||
/// `skipped`, `failed`).
|
||||
const METRIC_SWEEP_KEYS_TOTAL: &str = "rustfs_kms_deletion_sweep_keys_total";
|
||||
|
||||
/// Register metric descriptions once per process.
|
||||
fn describe_metrics() {
|
||||
static DESCRIBE: std::sync::Once = std::sync::Once::new();
|
||||
DESCRIBE.call_once(|| {
|
||||
metrics::describe_gauge!(
|
||||
METRIC_PENDING_DELETION_KEYS,
|
||||
"KMS keys scheduled for deletion whose deadline has not passed yet"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
METRIC_TOMBSTONE_KEYS,
|
||||
"KMS keys left tombstoned by an interrupted removal, still awaiting the deletion sweep"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
METRIC_OLDEST_ROTATION_AGE_SECONDS,
|
||||
"Seconds since the least recently rotated usable KMS key was last rotated, counting from creation for keys that were never rotated"
|
||||
);
|
||||
metrics::describe_counter!(METRIC_SWEEP_KEYS_TOTAL, "Total keys acted on by the KMS deletion sweep, by outcome");
|
||||
});
|
||||
}
|
||||
|
||||
/// Aggregate view of the key set, accumulated while the sweep pages through it.
|
||||
///
|
||||
/// Counts and one maximum age only: these feed label-less gauges, so no key
|
||||
/// identifier can reach the metric stream through them.
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq)]
|
||||
struct KeyCensus {
|
||||
pending_deletion: usize,
|
||||
tombstones: usize,
|
||||
/// Longest time since last rotation across keys that are still usable,
|
||||
/// counting from creation for keys that were never rotated. Keys on their
|
||||
/// way out are excluded: they will never be rotated again, and would
|
||||
/// otherwise pin the gauge high until the sweep finishes removing them.
|
||||
oldest_rotation_age_seconds: f64,
|
||||
}
|
||||
|
||||
impl KeyCensus {
|
||||
fn observe(&mut self, key: &KeyInfo, now: &Zoned) {
|
||||
match key.status {
|
||||
KeyStatus::PendingDeletion => self.pending_deletion += 1,
|
||||
KeyStatus::Deleted => self.tombstones += 1,
|
||||
KeyStatus::Active | KeyStatus::Disabled => {
|
||||
let rotated_at = key.rotated_at.as_ref().unwrap_or(&key.created_at);
|
||||
self.oldest_rotation_age_seconds = self.oldest_rotation_age_seconds.max(seconds_between(rotated_at, now));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Seconds from `earlier` to `now`, clamped at zero so clock skew (or a
|
||||
/// timestamp persisted by a node running ahead) cannot produce a negative age.
|
||||
fn seconds_between(earlier: &Zoned, now: &Zoned) -> f64 {
|
||||
(now.timestamp().as_second() - earlier.timestamp().as_second()).max(0) as f64
|
||||
}
|
||||
|
||||
/// Publish one sweep's counters, plus the lifecycle gauges when `census` covers
|
||||
/// the whole key set.
|
||||
fn record_sweep(report: &SweepReport, census: Option<KeyCensus>) {
|
||||
describe_metrics();
|
||||
for (outcome, count) in [
|
||||
("removed", report.removed.len()),
|
||||
("blocked", report.blocked.len()),
|
||||
("skipped", report.skipped),
|
||||
("failed", report.failed),
|
||||
] {
|
||||
// Emitted even at zero so every outcome series exists from the first
|
||||
// sweep on and a rate over it is defined.
|
||||
metrics::counter!(METRIC_SWEEP_KEYS_TOTAL, "outcome" => outcome).increment(count as u64);
|
||||
}
|
||||
|
||||
// A sweep that could not finish listing saw only part of the key set;
|
||||
// publishing its counts would understate every gauge, so the previous
|
||||
// (complete) values are left standing instead.
|
||||
let Some(census) = census else { return };
|
||||
metrics::gauge!(METRIC_PENDING_DELETION_KEYS).set(census.pending_deletion as f64);
|
||||
metrics::gauge!(METRIC_TOMBSTONE_KEYS).set(census.tombstones as f64);
|
||||
metrics::gauge!(METRIC_OLDEST_ROTATION_AGE_SECONDS).set(census.oldest_rotation_age_seconds);
|
||||
}
|
||||
|
||||
/// Reports configuration that still references a KMS key.
|
||||
///
|
||||
/// Consulted before any material is destroyed; a non-empty result blocks the
|
||||
@@ -173,8 +68,6 @@ pub(crate) struct DeletionWorker {
|
||||
default_key_id: Option<String>,
|
||||
reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
|
||||
interval: Duration,
|
||||
backend_kind: &'static str,
|
||||
audit_sink: Option<Arc<dyn KmsAuditSink>>,
|
||||
}
|
||||
|
||||
impl DeletionWorker {
|
||||
@@ -182,24 +75,15 @@ impl DeletionWorker {
|
||||
backend: Arc<dyn KmsBackend>,
|
||||
default_key_id: Option<String>,
|
||||
reference_checker: Option<Arc<dyn DeletionReferenceChecker>>,
|
||||
backend_kind: &'static str,
|
||||
) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
default_key_id,
|
||||
reference_checker,
|
||||
backend_kind,
|
||||
audit_sink: None,
|
||||
interval: DEFAULT_SWEEP_INTERVAL,
|
||||
}
|
||||
}
|
||||
|
||||
/// Audit each irreversible key removal to `sink`.
|
||||
pub(crate) fn with_audit_sink(mut self, sink: Option<Arc<dyn KmsAuditSink>>) -> Self {
|
||||
self.audit_sink = sink;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move { self.run(cancel).await })
|
||||
}
|
||||
@@ -230,15 +114,10 @@ impl DeletionWorker {
|
||||
|
||||
/// Run one sweep at the given time. Exposed separately so tests can drive
|
||||
/// the expiry logic deterministically.
|
||||
///
|
||||
/// Also feeds the lifecycle gauges: every key it lists is counted into a
|
||||
/// [`KeyCensus`] on the way past, so the observability comes out of the
|
||||
/// pages the sweep already had to fetch.
|
||||
pub(crate) async fn sweep(&self, now: &Zoned) -> SweepReport {
|
||||
let mut report = SweepReport::default();
|
||||
let mut census = KeyCensus::default();
|
||||
let mut marker: Option<String> = None;
|
||||
let listed_everything = loop {
|
||||
loop {
|
||||
let request = ListKeysRequest {
|
||||
limit: Some(100),
|
||||
marker: marker.clone(),
|
||||
@@ -250,99 +129,54 @@ impl DeletionWorker {
|
||||
Err(error) => {
|
||||
warn!(%error, "KMS deletion sweep could not list keys");
|
||||
report.failed += 1;
|
||||
break false;
|
||||
return report;
|
||||
}
|
||||
};
|
||||
for key in &response.keys {
|
||||
// Keys this sweep destroys are left out of the census: the
|
||||
// gauges describe the key set as it stands once the sweep is
|
||||
// done, not as it was when the page was listed.
|
||||
let removed = if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
|
||||
self.process_key(&key.key_id, now, &mut report).await
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if !removed {
|
||||
census.observe(key, now);
|
||||
if matches!(key.status, KeyStatus::PendingDeletion | KeyStatus::Deleted) {
|
||||
self.process_key(&key.key_id, now, &mut report).await;
|
||||
}
|
||||
}
|
||||
if !response.truncated {
|
||||
break true;
|
||||
break;
|
||||
}
|
||||
match response.next_marker {
|
||||
Some(next_marker) => marker = Some(next_marker),
|
||||
// Truncated without a marker: the rest of the key set is out
|
||||
// of reach, so the census is incomplete.
|
||||
None => break false,
|
||||
None => break,
|
||||
}
|
||||
};
|
||||
record_sweep(&report, listed_everything.then_some(census));
|
||||
}
|
||||
report
|
||||
}
|
||||
|
||||
/// Handle one key that is on its way out; reports whether it was removed.
|
||||
async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) -> bool {
|
||||
async fn process_key(&self, key_id: &str, now: &Zoned, report: &mut SweepReport) {
|
||||
// Never remove a key that live configuration still points at. The
|
||||
// default key check is built in; broader references (bucket
|
||||
// encryption settings, ...) come from the injected checker.
|
||||
if self.default_key_id.as_deref() == Some(key_id) {
|
||||
warn!(key_id, "expired KMS key is still the default key; refusing removal");
|
||||
report.blocked.push(key_id.to_string());
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
if let Some(checker) = &self.reference_checker {
|
||||
let references = checker.references(key_id).await;
|
||||
if !references.is_empty() {
|
||||
warn!(key_id, ?references, "expired KMS key is still referenced; refusing removal");
|
||||
report.blocked.push(key_id.to_string());
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// The backend re-checks state and deadline under its own write
|
||||
// synchronization, so a cancellation racing this sweep wins there.
|
||||
let started = Instant::now();
|
||||
let outcome = self.backend.remove_expired_key(key_id, now).await;
|
||||
match &outcome {
|
||||
Ok(ExpiredKeyRemoval::Removed) => {
|
||||
report.removed.push(key_id.to_string());
|
||||
self.audit_removal(key_id, started, &outcome);
|
||||
true
|
||||
}
|
||||
Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => {
|
||||
report.skipped += 1;
|
||||
false
|
||||
}
|
||||
match self.backend.remove_expired_key(key_id, now).await {
|
||||
Ok(ExpiredKeyRemoval::Removed) => report.removed.push(key_id.to_string()),
|
||||
Ok(ExpiredKeyRemoval::StateChanged | ExpiredKeyRemoval::NotExpired) => report.skipped += 1,
|
||||
Err(error) => {
|
||||
warn!(key_id, %error, "failed to remove expired KMS key; will retry next sweep");
|
||||
report.failed += 1;
|
||||
self.audit_removal(key_id, started, &outcome);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Record an attempted destruction of key material.
|
||||
///
|
||||
/// Only attempts that reached the backend are audited: a key the sweep
|
||||
/// declined to touch (not yet due, still referenced, state changed under
|
||||
/// us) was never at risk, and recording it as a deletion event would
|
||||
/// drown the real ones.
|
||||
fn audit_removal(&self, key_id: &str, started: Instant, outcome: &Result<ExpiredKeyRemoval>) {
|
||||
let Some(sink) = self.audit_sink.as_ref() else {
|
||||
return;
|
||||
};
|
||||
|
||||
// Removal runs on a background sweep, so there is no request
|
||||
// principal to attribute it to.
|
||||
let context = OperationContext::internal();
|
||||
sink.emit(
|
||||
KmsAuditRecord::new(KmsAuditOperation::DeleteKey, &context, self.backend_kind)
|
||||
.with_key_id(Some(key_id))
|
||||
.with_latency(started.elapsed())
|
||||
.with_result(outcome),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -382,7 +216,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn worker(backend: Arc<LocalKmsBackend>) -> DeletionWorker {
|
||||
DeletionWorker::new(backend, None, None, "local")
|
||||
DeletionWorker::new(backend, None, None)
|
||||
}
|
||||
|
||||
fn after_window() -> Zoned {
|
||||
@@ -473,7 +307,7 @@ mod tests {
|
||||
schedule(&backend, &key_id).await;
|
||||
|
||||
// Blocked while it is the configured default key.
|
||||
let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None, "local");
|
||||
let as_default = DeletionWorker::new(backend.clone(), Some(key_id.clone()), None);
|
||||
let report = as_default.sweep(&after_window()).await;
|
||||
assert_eq!(report.blocked, vec![key_id.clone()]);
|
||||
assert!(report.removed.is_empty());
|
||||
@@ -483,7 +317,6 @@ mod tests {
|
||||
backend.clone(),
|
||||
None,
|
||||
Some(Arc::new(StaticReferences(vec!["bucket:sse-bucket".to_string()]))),
|
||||
"local",
|
||||
);
|
||||
let report = with_references.sweep(&after_window()).await;
|
||||
assert_eq!(report.blocked, vec![key_id.clone()]);
|
||||
@@ -494,53 +327,11 @@ mod tests {
|
||||
.expect("blocked key must still exist");
|
||||
|
||||
// Removed once nothing references it anymore.
|
||||
let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))), "local");
|
||||
let unreferenced = DeletionWorker::new(backend.clone(), None, Some(Arc::new(StaticReferences(Vec::new()))));
|
||||
let report = unreferenced.sweep(&after_window()).await;
|
||||
assert_eq!(report.removed, vec![key_id.clone()]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn only_attempted_removals_are_audited() {
|
||||
#[derive(Default)]
|
||||
struct CapturingSink {
|
||||
records: std::sync::Mutex<Vec<KmsAuditRecord>>,
|
||||
}
|
||||
|
||||
impl KmsAuditSink for CapturingSink {
|
||||
fn emit(&self, record: KmsAuditRecord) {
|
||||
self.records.lock().expect("sink lock should not be poisoned").push(record);
|
||||
}
|
||||
}
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("temp dir");
|
||||
let backend = local_backend(&temp_dir).await;
|
||||
let key_id = create_key(&backend, "audited-removal").await;
|
||||
schedule(&backend, &key_id).await;
|
||||
|
||||
let sink = Arc::new(CapturingSink::default());
|
||||
let worker = DeletionWorker::new(backend.clone(), None, None, "local").with_audit_sink(Some(sink.clone()));
|
||||
|
||||
// A key that is not yet due was never at risk, so it produces no record.
|
||||
worker.sweep(&Zoned::now()).await;
|
||||
assert!(
|
||||
sink.records.lock().expect("sink lock").is_empty(),
|
||||
"a key the sweep declined to touch must not be audited as a deletion"
|
||||
);
|
||||
|
||||
worker.sweep(&after_window()).await;
|
||||
|
||||
let records = sink.records.lock().expect("sink lock");
|
||||
assert_eq!(records.len(), 1, "the removal should be audited exactly once");
|
||||
let record = &records[0];
|
||||
assert_eq!(record.operation, crate::audit::KmsAuditOperation::DeleteKey);
|
||||
assert_eq!(record.event, rustfs_s3_types::EventName::KmsKeyDeleted);
|
||||
assert_eq!(record.outcome, crate::audit::KmsAuditOutcome::Success);
|
||||
assert_eq!(record.key_id.as_deref(), Some(key_id.as_str()));
|
||||
assert_eq!(record.backend, "local");
|
||||
// Background work has no request principal to attribute.
|
||||
assert_eq!(record.principal, OperationContext::INTERNAL_PRINCIPAL);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn deadline_survives_backend_restart_and_sweep_completes_it() {
|
||||
let temp_dir = tempfile::tempdir().expect("temp dir");
|
||||
@@ -604,156 +395,4 @@ mod tests {
|
||||
cancel.cancel();
|
||||
task.await.expect("worker task must stop after cancellation");
|
||||
}
|
||||
|
||||
// -- Metric emission ----------------------------------------------------
|
||||
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
|
||||
type MetricEntry = (
|
||||
metrics_util::CompositeKey,
|
||||
Option<metrics::Unit>,
|
||||
Option<metrics::SharedString>,
|
||||
DebugValue,
|
||||
);
|
||||
|
||||
/// Run `test` on a current-thread runtime under a debugging recorder and
|
||||
/// return one snapshot of everything it emitted.
|
||||
///
|
||||
/// A single snapshot per test on purpose: `Snapshotter::snapshot` drains
|
||||
/// the recorded state, so taking it per assertion would only show the
|
||||
/// first assertion any data.
|
||||
fn record_metrics<Out>(
|
||||
test: impl FnOnce() -> std::pin::Pin<Box<dyn std::future::Future<Output = Out>>>,
|
||||
) -> (Vec<MetricEntry>, Out) {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let out = metrics::with_local_recorder(&recorder, || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("current-thread runtime must build");
|
||||
runtime.block_on(test())
|
||||
});
|
||||
(snapshotter.snapshot().into_vec(), out)
|
||||
}
|
||||
|
||||
fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
|
||||
snapshot.iter().find_map(|(composite, _unit, _description, value)| {
|
||||
let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Gauge(value)) => Some(value.into_inner()),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn counter_value(snapshot: &[MetricEntry], name: &str, outcome: &str) -> u64 {
|
||||
snapshot
|
||||
.iter()
|
||||
.filter_map(|(composite, _unit, _description, value)| {
|
||||
let matches = composite.kind() == MetricKind::Counter
|
||||
&& composite.key().name() == name
|
||||
&& composite
|
||||
.key()
|
||||
.labels()
|
||||
.any(|label| label.key() == "outcome" && label.value() == outcome);
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Counter(count)) => Some(*count),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn key_info(key_id: &str, status: KeyStatus, created_at: Zoned, rotated_at: Option<Zoned>) -> KeyInfo {
|
||||
KeyInfo {
|
||||
key_id: key_id.to_string(),
|
||||
description: None,
|
||||
algorithm: "AES-256".to_string(),
|
||||
usage: KeyUsage::EncryptDecrypt,
|
||||
status,
|
||||
version: 1,
|
||||
metadata: std::collections::HashMap::new(),
|
||||
tags: std::collections::HashMap::new(),
|
||||
created_at,
|
||||
rotated_at,
|
||||
created_by: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn census_ages_from_rotation_and_ignores_departing_keys() {
|
||||
let now = Zoned::now();
|
||||
let day = Duration::from_secs(86400);
|
||||
let mut census = KeyCensus::default();
|
||||
|
||||
// Rotation beats creation as the age baseline...
|
||||
census.observe(
|
||||
&key_info("rotated", KeyStatus::Active, now.clone() - 30 * day, Some(now.clone() - 2 * day)),
|
||||
&now,
|
||||
);
|
||||
// ...and a key that was never rotated ages from its creation.
|
||||
census.observe(&key_info("never-rotated", KeyStatus::Disabled, now.clone() - 5 * day, None), &now);
|
||||
// Keys on their way out only ever move the counts: they will not be
|
||||
// rotated again, so their age must not drive the rotation gauge.
|
||||
census.observe(&key_info("pending", KeyStatus::PendingDeletion, now.clone() - 400 * day, None), &now);
|
||||
census.observe(&key_info("tombstone", KeyStatus::Deleted, now.clone() - 400 * day, None), &now);
|
||||
|
||||
assert_eq!(census.pending_deletion, 1);
|
||||
assert_eq!(census.tombstones, 1);
|
||||
let expected = 5.0 * 86400.0;
|
||||
assert!(
|
||||
(census.oldest_rotation_age_seconds - expected).abs() < 1.0,
|
||||
"expected the never-rotated key to set the age, got {}",
|
||||
census.oldest_rotation_age_seconds
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sweep_publishes_lifecycle_gauges_without_key_labels() {
|
||||
let (snapshot, key_ids) = record_metrics(|| {
|
||||
Box::pin(async {
|
||||
let temp_dir = tempfile::tempdir().expect("temp dir");
|
||||
let backend = local_backend(&temp_dir).await;
|
||||
let live = create_key(&backend, "live-key").await;
|
||||
let doomed = create_key(&backend, "doomed-key").await;
|
||||
schedule(&backend, &doomed).await;
|
||||
|
||||
// Three days in, still inside the seven-day window: the sweep
|
||||
// observes the scheduled key instead of removing it.
|
||||
let report = worker(backend.clone())
|
||||
.sweep(&(Zoned::now() + Duration::from_secs(3 * 86400)))
|
||||
.await;
|
||||
assert_eq!(report.skipped, 1);
|
||||
assert!(report.removed.is_empty());
|
||||
vec![live, doomed]
|
||||
})
|
||||
});
|
||||
|
||||
assert_eq!(gauge_value(&snapshot, METRIC_PENDING_DELETION_KEYS), Some(1.0));
|
||||
assert_eq!(gauge_value(&snapshot, METRIC_TOMBSTONE_KEYS), Some(0.0));
|
||||
let age = gauge_value(&snapshot, METRIC_OLDEST_ROTATION_AGE_SECONDS).expect("rotation age gauge must be published");
|
||||
// Both keys were just created, and only the usable one counts, so the
|
||||
// reported age is the sweep's own offset into the future.
|
||||
assert!(
|
||||
(age - 3.0 * 86400.0).abs() < 60.0,
|
||||
"expected the sweep offset as the rotation age, got {age}"
|
||||
);
|
||||
assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "skipped"), 1);
|
||||
assert_eq!(counter_value(&snapshot, METRIC_SWEEP_KEYS_TOTAL, "removed"), 0);
|
||||
|
||||
for (composite, ..) in &snapshot {
|
||||
for label in composite.key().labels() {
|
||||
for key_id in &key_ids {
|
||||
assert!(
|
||||
!label.value().contains(key_id.as_str()),
|
||||
"metric {} leaked a key identifier through label {}",
|
||||
composite.key().name(),
|
||||
label.key()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,7 +65,6 @@
|
||||
|
||||
// Core modules
|
||||
pub mod api_types;
|
||||
pub mod audit;
|
||||
pub mod backends;
|
||||
pub mod backup;
|
||||
mod cache;
|
||||
@@ -75,7 +74,6 @@ mod encryption;
|
||||
mod error;
|
||||
pub mod manager;
|
||||
mod policy;
|
||||
pub mod probe;
|
||||
pub mod service;
|
||||
pub mod service_manager;
|
||||
mod time_serde;
|
||||
@@ -88,14 +86,11 @@ pub use api_types::{
|
||||
StartKmsResponse, StopKmsResponse, TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse,
|
||||
UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
|
||||
};
|
||||
pub use audit::{KmsAuditOperation, KmsAuditOutcome, KmsAuditRecord, KmsAuditSink, redact_encryption_context};
|
||||
pub use cache::KmsCacheStats;
|
||||
pub use config::*;
|
||||
pub use deletion_worker::DeletionReferenceChecker;
|
||||
pub use encryption::is_data_key_envelope;
|
||||
pub use error::{KmsError, KmsUnavailableError, Result};
|
||||
pub use manager::KmsManager;
|
||||
pub use probe::{ProbeFailureKind, ProbeResult, ProbeStatus};
|
||||
pub use service::{DataKey, ObjectEncryptionService};
|
||||
pub use service_manager::{
|
||||
KmsServiceManager, KmsServiceStatus, KmsStartOutcome, get_global_encryption_service, get_global_kms_service_manager,
|
||||
|
||||
+16
-669
@@ -14,19 +14,16 @@
|
||||
|
||||
//! KMS manager for handling key operations and backend coordination
|
||||
|
||||
use crate::audit::{KmsAuditOperation, KmsAuditRecord, KmsAuditSink};
|
||||
use crate::backends::KmsBackend;
|
||||
use crate::cache::{KmsCache, KmsCacheStats};
|
||||
use crate::cache::KmsCache;
|
||||
use crate::config::KmsConfig;
|
||||
use crate::error::Result;
|
||||
use crate::types::{
|
||||
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyRequest, CreateKeyResponse, DecryptRequest, DecryptResponse,
|
||||
DeleteKeyRequest, DeleteKeyResponse, DescribeKeyRequest, DescribeKeyResponse, EncryptRequest, EncryptResponse,
|
||||
GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse, OperationContext,
|
||||
GenerateDataKeyRequest, GenerateDataKeyResponse, ListKeysRequest, ListKeysResponse,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// KMS Manager coordinates operations between backends and caching
|
||||
@@ -36,8 +33,6 @@ pub struct KmsManager {
|
||||
cache: Arc<RwLock<KmsCache>>,
|
||||
default_key_id: Option<String>,
|
||||
enable_cache: bool,
|
||||
backend_kind: &'static str,
|
||||
audit_sink: Option<Arc<dyn KmsAuditSink>>,
|
||||
}
|
||||
|
||||
impl KmsManager {
|
||||
@@ -49,72 +44,16 @@ impl KmsManager {
|
||||
cache,
|
||||
default_key_id: config.default_key_id,
|
||||
enable_cache: config.enable_cache,
|
||||
backend_kind: config.backend.as_str(),
|
||||
audit_sink: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Send an audit record for every management operation to `sink`.
|
||||
///
|
||||
/// Without a sink the manager builds no records at all, so a deployment
|
||||
/// that does not consume KMS audit records is unaffected.
|
||||
pub fn with_audit_sink(mut self, sink: Arc<dyn KmsAuditSink>) -> Self {
|
||||
self.audit_sink = Some(sink);
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the default key ID if configured
|
||||
pub fn get_default_key_id(&self) -> Option<&String> {
|
||||
self.default_key_id.as_ref()
|
||||
}
|
||||
|
||||
/// Emit an audit record for a completed management operation.
|
||||
///
|
||||
/// Called after the operation resolved, so nothing here can change its
|
||||
/// result; the record is built only when a sink is installed.
|
||||
fn audit<T>(
|
||||
&self,
|
||||
operation: KmsAuditOperation,
|
||||
context: &OperationContext,
|
||||
key_id: Option<&str>,
|
||||
started: Instant,
|
||||
result: &Result<T>,
|
||||
) {
|
||||
let Some(sink) = self.audit_sink.as_ref() else {
|
||||
return;
|
||||
};
|
||||
|
||||
sink.emit(
|
||||
KmsAuditRecord::new(operation, context, self.backend_kind)
|
||||
.with_key_id(key_id)
|
||||
.with_latency(started.elapsed())
|
||||
.with_result(result),
|
||||
);
|
||||
}
|
||||
|
||||
/// Create a new master key
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::create_key_with_context`].
|
||||
pub async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
|
||||
self.create_key_with_context(request, &OperationContext::internal()).await
|
||||
}
|
||||
|
||||
/// Create a new master key on behalf of `context`'s principal
|
||||
pub async fn create_key_with_context(
|
||||
&self,
|
||||
request: CreateKeyRequest,
|
||||
context: &OperationContext,
|
||||
) -> Result<CreateKeyResponse> {
|
||||
let started = Instant::now();
|
||||
let key_name = request.key_name.clone();
|
||||
let result = self.create_key_inner(request).await;
|
||||
let key_id = result.as_ref().ok().map(|r| r.key_id.as_str()).or(key_name.as_deref());
|
||||
self.audit(KmsAuditOperation::CreateKey, context, key_id, started, &result);
|
||||
result
|
||||
}
|
||||
|
||||
async fn create_key_inner(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
|
||||
let response = self.backend.create_key(request).await?;
|
||||
|
||||
// Cache the key metadata if enabled
|
||||
@@ -145,27 +84,7 @@ impl KmsManager {
|
||||
}
|
||||
|
||||
/// Describe a key
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::describe_key_with_context`].
|
||||
pub async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
|
||||
self.describe_key_with_context(request, &OperationContext::internal()).await
|
||||
}
|
||||
|
||||
/// Describe a key on behalf of `context`'s principal
|
||||
pub async fn describe_key_with_context(
|
||||
&self,
|
||||
request: DescribeKeyRequest,
|
||||
context: &OperationContext,
|
||||
) -> Result<DescribeKeyResponse> {
|
||||
let started = Instant::now();
|
||||
let key_id = request.key_id.clone();
|
||||
let result = self.describe_key_inner(request).await;
|
||||
self.audit(KmsAuditOperation::DescribeKey, context, Some(&key_id), started, &result);
|
||||
result
|
||||
}
|
||||
|
||||
async fn describe_key_inner(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
|
||||
// Check cache first if enabled
|
||||
if self.enable_cache {
|
||||
let cache = self.cache.read().await;
|
||||
@@ -190,24 +109,12 @@ impl KmsManager {
|
||||
}
|
||||
|
||||
/// List keys
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::list_keys_with_context`].
|
||||
pub async fn list_keys(&self, request: ListKeysRequest) -> Result<ListKeysResponse> {
|
||||
self.list_keys_with_context(request, &OperationContext::internal()).await
|
||||
self.backend.list_keys(request).await
|
||||
}
|
||||
|
||||
/// List keys on behalf of `context`'s principal
|
||||
pub async fn list_keys_with_context(&self, request: ListKeysRequest, context: &OperationContext) -> Result<ListKeysResponse> {
|
||||
let started = Instant::now();
|
||||
let result = self.backend.list_keys(request).await;
|
||||
// Listing spans keys, so the record carries no key id.
|
||||
self.audit(KmsAuditOperation::ListKeys, context, None, started, &result);
|
||||
result
|
||||
}
|
||||
|
||||
/// Get cache statistics, or `None` when caching is disabled
|
||||
pub async fn cache_stats(&self) -> Option<KmsCacheStats> {
|
||||
/// Get cache statistics
|
||||
pub async fn cache_stats(&self) -> Option<(u64, u64)> {
|
||||
if self.enable_cache {
|
||||
let cache = self.cache.read().await;
|
||||
Some(cache.stats())
|
||||
@@ -226,27 +133,7 @@ impl KmsManager {
|
||||
}
|
||||
|
||||
/// Delete a key
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::delete_key_with_context`].
|
||||
pub async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
|
||||
self.delete_key_with_context(request, &OperationContext::internal()).await
|
||||
}
|
||||
|
||||
/// Delete a key on behalf of `context`'s principal
|
||||
pub async fn delete_key_with_context(
|
||||
&self,
|
||||
request: DeleteKeyRequest,
|
||||
context: &OperationContext,
|
||||
) -> Result<DeleteKeyResponse> {
|
||||
let started = Instant::now();
|
||||
let key_id = request.key_id.clone();
|
||||
let result = self.delete_key_inner(request).await;
|
||||
self.audit(KmsAuditOperation::ScheduleKeyDeletion, context, Some(&key_id), started, &result);
|
||||
result
|
||||
}
|
||||
|
||||
async fn delete_key_inner(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
|
||||
let response = self.backend.delete_key(request).await?;
|
||||
|
||||
// Remove from cache if enabled and key is being deleted
|
||||
@@ -259,28 +146,7 @@ impl KmsManager {
|
||||
}
|
||||
|
||||
/// Cancel key deletion
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::cancel_key_deletion_with_context`].
|
||||
pub async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
|
||||
self.cancel_key_deletion_with_context(request, &OperationContext::internal())
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cancel key deletion on behalf of `context`'s principal
|
||||
pub async fn cancel_key_deletion_with_context(
|
||||
&self,
|
||||
request: CancelKeyDeletionRequest,
|
||||
context: &OperationContext,
|
||||
) -> Result<CancelKeyDeletionResponse> {
|
||||
let started = Instant::now();
|
||||
let key_id = request.key_id.clone();
|
||||
let result = self.cancel_key_deletion_inner(request).await;
|
||||
self.audit(KmsAuditOperation::CancelKeyDeletion, context, Some(&key_id), started, &result);
|
||||
result
|
||||
}
|
||||
|
||||
async fn cancel_key_deletion_inner(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
|
||||
let response = self.backend.cancel_key_deletion(request).await?;
|
||||
|
||||
// Update cache if enabled
|
||||
@@ -293,79 +159,22 @@ impl KmsManager {
|
||||
}
|
||||
|
||||
/// Enable a disabled key
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::enable_key_with_context`].
|
||||
pub async fn enable_key(&self, key_id: &str) -> Result<()> {
|
||||
self.enable_key_with_context(key_id, &OperationContext::internal()).await
|
||||
}
|
||||
|
||||
/// Enable a disabled key on behalf of `context`'s principal
|
||||
pub async fn enable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
|
||||
let started = Instant::now();
|
||||
let result = self.backend.enable_key(key_id).await;
|
||||
if result.is_ok() {
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
}
|
||||
self.audit(KmsAuditOperation::EnableKey, context, Some(key_id), started, &result);
|
||||
result
|
||||
self.backend.enable_key(key_id).await?;
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Disable a key; existing data remains decryptable
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::disable_key_with_context`].
|
||||
pub async fn disable_key(&self, key_id: &str) -> Result<()> {
|
||||
self.disable_key_with_context(key_id, &OperationContext::internal()).await
|
||||
}
|
||||
|
||||
/// Disable a key on behalf of `context`'s principal
|
||||
pub async fn disable_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
|
||||
let started = Instant::now();
|
||||
let result = self.backend.disable_key(key_id).await;
|
||||
if result.is_ok() {
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
}
|
||||
self.audit(KmsAuditOperation::DisableKey, context, Some(key_id), started, &result);
|
||||
result
|
||||
self.backend.disable_key(key_id).await?;
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rotate a key to a new version
|
||||
///
|
||||
/// Audited as an internal operation; callers serving an authenticated
|
||||
/// request should use [`Self::rotate_key_with_context`].
|
||||
pub async fn rotate_key(&self, key_id: &str) -> Result<()> {
|
||||
self.rotate_key_with_context(key_id, &OperationContext::internal()).await
|
||||
}
|
||||
|
||||
/// Rotate a key on behalf of `context`'s principal
|
||||
pub async fn rotate_key_with_context(&self, key_id: &str, context: &OperationContext) -> Result<()> {
|
||||
let started = Instant::now();
|
||||
let result = self.backend.rotate_key(key_id).await;
|
||||
if result.is_ok() {
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
}
|
||||
self.audit(KmsAuditOperation::RotateKey, context, Some(key_id), started, &result);
|
||||
result
|
||||
}
|
||||
|
||||
/// Replace a key's description; `None` clears it
|
||||
pub async fn update_key_description(&self, key_id: &str, description: Option<&str>) -> Result<()> {
|
||||
self.backend.update_key_description(key_id, description).await?;
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Add or overwrite key tags, leaving every other tag untouched
|
||||
pub async fn tag_key(&self, key_id: &str, tags: &HashMap<String, String>) -> Result<()> {
|
||||
self.backend.tag_key(key_id, tags).await?;
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove key tags; tags that are not set are ignored
|
||||
pub async fn untag_key(&self, key_id: &str, tag_keys: &[String]) -> Result<()> {
|
||||
self.backend.untag_key(key_id, tag_keys).await?;
|
||||
self.backend.rotate_key(key_id).await?;
|
||||
self.invalidate_cached_metadata(key_id).await;
|
||||
Ok(())
|
||||
}
|
||||
@@ -399,470 +208,11 @@ impl KmsManager {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::audit::KmsAuditOutcome;
|
||||
use crate::backends::local::LocalKmsBackend;
|
||||
use crate::error::KmsError;
|
||||
use crate::types::{KeyMetadata, KeySpec, KeyState, KeyUsage};
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine as _;
|
||||
use jiff::Zoned;
|
||||
use crate::types::{KeySpec, KeyState, KeyUsage};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
use tempfile::tempdir;
|
||||
|
||||
/// Sink that keeps every record so tests can assert on the audit trail.
|
||||
#[derive(Default)]
|
||||
struct CapturingSink {
|
||||
records: Mutex<Vec<KmsAuditRecord>>,
|
||||
}
|
||||
|
||||
impl KmsAuditSink for CapturingSink {
|
||||
fn emit(&self, record: KmsAuditRecord) {
|
||||
self.records
|
||||
.lock()
|
||||
.expect("audit records lock should not be poisoned")
|
||||
.push(record);
|
||||
}
|
||||
}
|
||||
|
||||
impl CapturingSink {
|
||||
fn records(&self) -> Vec<KmsAuditRecord> {
|
||||
self.records
|
||||
.lock()
|
||||
.expect("audit records lock should not be poisoned")
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn take_one(&self) -> KmsAuditRecord {
|
||||
let mut records = self.records.lock().expect("audit records lock should not be poisoned");
|
||||
assert_eq!(records.len(), 1, "expected exactly one audit record, got {records:?}");
|
||||
records.remove(0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Backend whose management operations all succeed or all fail, so a
|
||||
/// single test can drive every audited operation down both paths —
|
||||
/// including operations no real backend supports on both.
|
||||
struct ScriptedBackend {
|
||||
failure: Option<KmsError>,
|
||||
}
|
||||
|
||||
impl ScriptedBackend {
|
||||
fn succeeding() -> Self {
|
||||
Self { failure: None }
|
||||
}
|
||||
|
||||
fn failing(failure: KmsError) -> Self {
|
||||
Self { failure: Some(failure) }
|
||||
}
|
||||
|
||||
fn check(&self) -> Result<()> {
|
||||
match &self.failure {
|
||||
Some(failure) => Err(failure.clone()),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn metadata(key_id: &str) -> KeyMetadata {
|
||||
KeyMetadata {
|
||||
key_id: key_id.to_string(),
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "RUSTFS_KMS".to_string(),
|
||||
key_manager: "RUSTFS".to_string(),
|
||||
tags: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl KmsBackend for ScriptedBackend {
|
||||
async fn create_key(&self, request: CreateKeyRequest) -> Result<CreateKeyResponse> {
|
||||
self.check()?;
|
||||
let key_id = request.key_name.unwrap_or_else(|| "scripted-key".to_string());
|
||||
Ok(CreateKeyResponse {
|
||||
key_metadata: Self::metadata(&key_id),
|
||||
key_id,
|
||||
})
|
||||
}
|
||||
|
||||
async fn encrypt(&self, _request: EncryptRequest) -> Result<EncryptResponse> {
|
||||
unimplemented!("data plane is not audited by the manager")
|
||||
}
|
||||
|
||||
async fn decrypt(&self, _request: DecryptRequest) -> Result<DecryptResponse> {
|
||||
unimplemented!("data plane is not audited by the manager")
|
||||
}
|
||||
|
||||
async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> Result<GenerateDataKeyResponse> {
|
||||
unimplemented!("data plane is not audited by the manager")
|
||||
}
|
||||
|
||||
async fn describe_key(&self, request: DescribeKeyRequest) -> Result<DescribeKeyResponse> {
|
||||
self.check()?;
|
||||
Ok(DescribeKeyResponse {
|
||||
key_metadata: Self::metadata(&request.key_id),
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_keys(&self, _request: ListKeysRequest) -> Result<ListKeysResponse> {
|
||||
self.check()?;
|
||||
Ok(ListKeysResponse {
|
||||
keys: Vec::new(),
|
||||
next_marker: None,
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
|
||||
async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
|
||||
self.check()?;
|
||||
Ok(DeleteKeyResponse {
|
||||
key_metadata: Self::metadata(&request.key_id),
|
||||
key_id: request.key_id,
|
||||
deletion_date: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn cancel_key_deletion(&self, request: CancelKeyDeletionRequest) -> Result<CancelKeyDeletionResponse> {
|
||||
self.check()?;
|
||||
Ok(CancelKeyDeletionResponse {
|
||||
key_metadata: Self::metadata(&request.key_id),
|
||||
key_id: request.key_id,
|
||||
})
|
||||
}
|
||||
|
||||
async fn enable_key(&self, _key_id: &str) -> Result<()> {
|
||||
self.check()
|
||||
}
|
||||
|
||||
async fn disable_key(&self, _key_id: &str) -> Result<()> {
|
||||
self.check()
|
||||
}
|
||||
|
||||
async fn rotate_key(&self, _key_id: &str) -> Result<()> {
|
||||
self.check()
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
const AUDITED_KEY_ID: &str = "audited-key";
|
||||
|
||||
/// Prefix length used when checking that a record embedded no *part* of a
|
||||
/// secret. Long enough that a collision with unrelated text is not a real
|
||||
/// concern.
|
||||
const FRAGMENT_LEN: usize = 24;
|
||||
|
||||
fn scripted_manager(backend: ScriptedBackend) -> (KmsManager, Arc<CapturingSink>) {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
|
||||
let sink = Arc::new(CapturingSink::default());
|
||||
let manager = KmsManager::new(Arc::new(backend), config).with_audit_sink(sink.clone());
|
||||
(manager, sink)
|
||||
}
|
||||
|
||||
fn request_context() -> OperationContext {
|
||||
OperationContext::new("arn:aws:iam::user/alice".to_string())
|
||||
.with_source_ip("192.0.2.10".to_string())
|
||||
.with_user_agent("rustfs-admin/1".to_string())
|
||||
.with_context("requestID".to_string(), "req-42".to_string())
|
||||
}
|
||||
|
||||
/// Drive one management operation and return the single record it emitted.
|
||||
async fn run_operation(manager: &KmsManager, operation: KmsAuditOperation, context: &OperationContext) -> Result<()> {
|
||||
match operation {
|
||||
KmsAuditOperation::CreateKey => manager
|
||||
.create_key_with_context(
|
||||
CreateKeyRequest {
|
||||
key_name: Some(AUDITED_KEY_ID.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
KmsAuditOperation::DescribeKey => manager
|
||||
.describe_key_with_context(
|
||||
DescribeKeyRequest {
|
||||
key_id: AUDITED_KEY_ID.to_string(),
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
KmsAuditOperation::ListKeys => manager
|
||||
.list_keys_with_context(ListKeysRequest::default(), context)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
KmsAuditOperation::ScheduleKeyDeletion => manager
|
||||
.delete_key_with_context(
|
||||
DeleteKeyRequest {
|
||||
key_id: AUDITED_KEY_ID.to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: None,
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
KmsAuditOperation::CancelKeyDeletion => manager
|
||||
.cancel_key_deletion_with_context(
|
||||
CancelKeyDeletionRequest {
|
||||
key_id: AUDITED_KEY_ID.to_string(),
|
||||
},
|
||||
context,
|
||||
)
|
||||
.await
|
||||
.map(|_| ()),
|
||||
KmsAuditOperation::EnableKey => manager.enable_key_with_context(AUDITED_KEY_ID, context).await,
|
||||
KmsAuditOperation::DisableKey => manager.disable_key_with_context(AUDITED_KEY_ID, context).await,
|
||||
KmsAuditOperation::RotateKey => manager.rotate_key_with_context(AUDITED_KEY_ID, context).await,
|
||||
// Physical removal happens on the background sweep, not here.
|
||||
KmsAuditOperation::DeleteKey => unreachable!("removal is audited by the deletion worker"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every management operation the manager serves, in audit terms.
|
||||
const AUDITED_OPERATIONS: [KmsAuditOperation; 8] = [
|
||||
KmsAuditOperation::CreateKey,
|
||||
KmsAuditOperation::DescribeKey,
|
||||
KmsAuditOperation::ListKeys,
|
||||
KmsAuditOperation::ScheduleKeyDeletion,
|
||||
KmsAuditOperation::CancelKeyDeletion,
|
||||
KmsAuditOperation::EnableKey,
|
||||
KmsAuditOperation::DisableKey,
|
||||
KmsAuditOperation::RotateKey,
|
||||
];
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_management_operation_emits_a_complete_success_record() {
|
||||
for operation in AUDITED_OPERATIONS {
|
||||
let (manager, sink) = scripted_manager(ScriptedBackend::succeeding());
|
||||
let context = request_context();
|
||||
|
||||
let outer = Instant::now();
|
||||
run_operation(&manager, operation, &context)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{} should succeed: {error}", operation.as_str()));
|
||||
let outer_elapsed = outer.elapsed();
|
||||
|
||||
let record = sink.take_one();
|
||||
assert_eq!(record.operation, operation);
|
||||
assert_eq!(record.event, operation.event_name());
|
||||
assert_eq!(record.outcome, KmsAuditOutcome::Success);
|
||||
assert_eq!(record.error_class, None);
|
||||
assert_eq!(record.operation_id, context.operation_id);
|
||||
assert_eq!(record.principal, "arn:aws:iam::user/alice");
|
||||
assert_eq!(record.source_ip.as_deref(), Some("192.0.2.10"));
|
||||
assert_eq!(record.user_agent.as_deref(), Some("rustfs-admin/1"));
|
||||
assert_eq!(record.backend, "local");
|
||||
assert_eq!(record.context.get("requestID").map(String::as_str), Some("req-42"));
|
||||
assert!(
|
||||
record.latency <= outer_elapsed,
|
||||
"{} reported a latency larger than the call it measured",
|
||||
operation.as_str()
|
||||
);
|
||||
|
||||
// Listing spans keys; every other operation names the key it touched.
|
||||
if operation == KmsAuditOperation::ListKeys {
|
||||
assert_eq!(record.key_id, None);
|
||||
} else {
|
||||
assert_eq!(record.key_id.as_deref(), Some(AUDITED_KEY_ID));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_management_operation_emits_a_failure_record() {
|
||||
for operation in AUDITED_OPERATIONS {
|
||||
let (manager, sink) = scripted_manager(ScriptedBackend::failing(KmsError::access_denied("denied by policy")));
|
||||
let context = request_context();
|
||||
|
||||
let error = run_operation(&manager, operation, &context)
|
||||
.await
|
||||
.expect_err("scripted backend should reject the operation");
|
||||
assert!(matches!(error, KmsError::AccessDenied { .. }));
|
||||
|
||||
let record = sink.take_one();
|
||||
assert_eq!(record.operation, operation);
|
||||
assert_eq!(record.event, operation.event_name());
|
||||
assert_eq!(record.outcome, KmsAuditOutcome::Failure);
|
||||
assert_eq!(record.error_class, Some("access_denied"));
|
||||
assert_eq!(record.operation_id, context.operation_id);
|
||||
assert_eq!(record.principal, "arn:aws:iam::user/alice");
|
||||
assert_eq!(record.source_ip.as_deref(), Some("192.0.2.10"));
|
||||
|
||||
// A denied create still has to name the key the caller asked for,
|
||||
// otherwise the record cannot answer "what were they after".
|
||||
if operation != KmsAuditOperation::ListKeys {
|
||||
assert_eq!(record.key_id.as_deref(), Some(AUDITED_KEY_ID));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn operations_are_unaffected_when_no_sink_is_installed() {
|
||||
// The audit trail is optional; without a sink the manager must behave
|
||||
// exactly as it did before records existed.
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
|
||||
let manager = KmsManager::new(Arc::new(ScriptedBackend::succeeding()), config);
|
||||
|
||||
for operation in AUDITED_OPERATIONS {
|
||||
run_operation(&manager, operation, &request_context())
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{} should succeed without a sink: {error}", operation.as_str()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn context_free_calls_are_attributed_to_the_internal_principal() {
|
||||
// Callers that have no authenticated identity must still be
|
||||
// distinguishable from an identity we failed to record.
|
||||
let (manager, sink) = scripted_manager(ScriptedBackend::succeeding());
|
||||
|
||||
manager
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: AUDITED_KEY_ID.to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("describe should succeed");
|
||||
|
||||
let record = sink.take_one();
|
||||
assert_eq!(record.principal, OperationContext::INTERNAL_PRINCIPAL);
|
||||
assert_eq!(record.source_ip, None);
|
||||
assert_eq!(record.user_agent, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsupported_lifecycle_operations_are_audited_with_their_own_class() {
|
||||
// The local backend has no version history, so rotation is a capability
|
||||
// gap rather than a policy denial; the audit trail must say so.
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
|
||||
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
|
||||
let sink = Arc::new(CapturingSink::default());
|
||||
let manager = KmsManager::new(backend, config).with_audit_sink(sink.clone());
|
||||
|
||||
let key_id = manager
|
||||
.create_key_with_context(
|
||||
CreateKeyRequest {
|
||||
key_name: Some("rotate-me".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
&request_context(),
|
||||
)
|
||||
.await
|
||||
.expect("create should succeed")
|
||||
.key_id;
|
||||
|
||||
manager
|
||||
.rotate_key_with_context(&key_id, &request_context())
|
||||
.await
|
||||
.expect_err("local rotation must be rejected");
|
||||
|
||||
let records = sink.records();
|
||||
let rotate = records.last().expect("rotation should be audited");
|
||||
assert_eq!(rotate.operation, KmsAuditOperation::RotateKey);
|
||||
assert_eq!(rotate.outcome, KmsAuditOutcome::Failure);
|
||||
assert_eq!(rotate.error_class, Some("unsupported_capability"));
|
||||
assert_eq!(rotate.key_id.as_deref(), Some(key_id.as_str()));
|
||||
}
|
||||
|
||||
/// Negative assertion: no audit record may reproduce key material. Driven
|
||||
/// against the real local backend so the assertion covers whatever the
|
||||
/// backend actually hands back, not a hand-written stand-in.
|
||||
#[tokio::test]
|
||||
async fn audit_records_never_reproduce_key_material() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
|
||||
let backend = Arc::new(LocalKmsBackend::new(config.clone()).await.expect("Failed to create backend"));
|
||||
let sink = Arc::new(CapturingSink::default());
|
||||
let manager = KmsManager::new(backend, config).with_audit_sink(sink.clone());
|
||||
|
||||
let grant_token = "grant-token-cec4d4b5a1";
|
||||
let context = request_context();
|
||||
|
||||
let key_id = manager
|
||||
.create_key_with_context(
|
||||
CreateKeyRequest {
|
||||
key_name: Some("material-key".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
.expect("create should succeed")
|
||||
.key_id;
|
||||
|
||||
// Produce real key material, then keep driving the management plane so
|
||||
// any record built afterwards is covered by the assertions below.
|
||||
let data_key = manager
|
||||
.generate_data_key(GenerateDataKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
key_spec: KeySpec::Aes256,
|
||||
encryption_context: HashMap::from([("bucket".to_string(), "secrets".to_string())]),
|
||||
})
|
||||
.await
|
||||
.expect("data key generation should succeed");
|
||||
let decrypted = manager
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: data_key.ciphertext_blob.clone(),
|
||||
encryption_context: HashMap::from([("bucket".to_string(), "secrets".to_string())]),
|
||||
grant_tokens: vec![grant_token.to_string()],
|
||||
})
|
||||
.await
|
||||
.expect("decrypt should succeed");
|
||||
|
||||
manager
|
||||
.describe_key_with_context(DescribeKeyRequest { key_id: key_id.clone() }, &context)
|
||||
.await
|
||||
.expect("describe should succeed");
|
||||
manager
|
||||
.list_keys_with_context(ListKeysRequest::default(), &context)
|
||||
.await
|
||||
.expect("list should succeed");
|
||||
manager
|
||||
.disable_key_with_context(&key_id, &context)
|
||||
.await
|
||||
.expect("disable should succeed");
|
||||
manager
|
||||
.enable_key_with_context(&key_id, &context)
|
||||
.await
|
||||
.expect("enable should succeed");
|
||||
|
||||
let base64 = base64::engine::general_purpose::STANDARD;
|
||||
let encodings = |bytes: &[u8]| vec![hex::encode(bytes), base64.encode(bytes)];
|
||||
let mut forbidden = vec![grant_token.to_string()];
|
||||
forbidden.extend(encodings(&data_key.plaintext_key));
|
||||
forbidden.extend(encodings(&decrypted.plaintext));
|
||||
forbidden.extend(encodings(&data_key.ciphertext_blob));
|
||||
// Fragments catch a record that embedded only part of a blob.
|
||||
let fragments: Vec<String> = forbidden
|
||||
.iter()
|
||||
.filter(|secret| secret.len() > FRAGMENT_LEN)
|
||||
.map(|secret| secret[..FRAGMENT_LEN].to_string())
|
||||
.collect();
|
||||
forbidden.extend(fragments);
|
||||
|
||||
let records = sink.records();
|
||||
assert!(!records.is_empty(), "management operations should have been audited");
|
||||
for record in &records {
|
||||
let rendered = format!("{record:?}");
|
||||
for secret in &forbidden {
|
||||
assert!(
|
||||
!rendered.contains(secret.as_str()),
|
||||
"audit record leaked key material or a grant token: {rendered}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_manager_operations() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
@@ -904,12 +254,9 @@ mod tests {
|
||||
let describe_response = manager.describe_key(describe_request).await.expect("Failed to describe key");
|
||||
assert_eq!(describe_response.key_metadata.key_id, create_response.key_id);
|
||||
|
||||
// Creating the key populated the cache, so the describe above was
|
||||
// served from it rather than from the backend.
|
||||
let stats = manager.cache_stats().await.expect("cache is enabled");
|
||||
assert_eq!(stats.entries, 1);
|
||||
assert_eq!(stats.hits, 1);
|
||||
assert_eq!(stats.misses, 0);
|
||||
// Test cache stats
|
||||
let stats = manager.cache_stats().await;
|
||||
assert!(stats.is_some());
|
||||
|
||||
// Test health check
|
||||
let health = manager.health_check().await.expect("Health check failed");
|
||||
|
||||
@@ -104,9 +104,7 @@ pub(crate) fn classify_vaultrs(error: &vaultrs::error::ClientError) -> ErrorClas
|
||||
}
|
||||
}
|
||||
|
||||
/// Retry classification of an HTTP status, shared by every backend that talks
|
||||
/// to an external KMS over HTTP.
|
||||
pub(crate) fn classify_status(code: u16) -> ErrorClass {
|
||||
fn classify_status(code: u16) -> ErrorClass {
|
||||
match code {
|
||||
429 | 500 | 502 | 503 | 504 => ErrorClass::RetryableStatus,
|
||||
_ => ErrorClass::Fatal,
|
||||
|
||||
@@ -1,894 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Background synthetic probe for the configured KMS backend.
|
||||
//!
|
||||
//! A health check that only reports "the backend answered" cannot distinguish a
|
||||
//! usable KMS from one that is reachable but no longer returns material that
|
||||
//! round-trips. Each probe round therefore generates a data key under a
|
||||
//! dedicated key, decrypts the returned ciphertext, and compares the plaintext,
|
||||
//! so a backend that answers without being usable becomes visible before object
|
||||
//! traffic depends on it.
|
||||
//!
|
||||
//! The probe key uses a fixed reserved id ([`PROBE_KEY_ID`]) and is created only
|
||||
//! when missing, so every node of a deployment converges on the same key and a
|
||||
//! create that loses the race against another node is a success, not an error.
|
||||
//!
|
||||
//! Backends that cannot host such a key — single-key read-only backends, or
|
||||
//! backends without data key generation — are reported as
|
||||
//! [`ProbeResult::Unsupported`] exactly once and then left alone: an
|
||||
//! unsupported backend is a configuration fact, not an outage, and must never
|
||||
//! raise failure metrics or alerts.
|
||||
//!
|
||||
//! Results are published two ways: as a lock-free [`ProbeStatus`] snapshot that
|
||||
//! readiness reporting can read without touching the backend, and as
|
||||
//! `rustfs_kms_probe_*` metrics. As everywhere else in this crate, metric labels
|
||||
//! carry only static outcome classes — never key identifiers, key material, or
|
||||
//! ciphertext.
|
||||
|
||||
use crate::backends::KmsBackend;
|
||||
use crate::error::KmsError;
|
||||
use crate::types::{CreateKeyRequest, DecryptRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec, KeyUsage};
|
||||
use arc_swap::ArcSwap;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use subtle::ConstantTimeEq;
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info, warn};
|
||||
use zeroize::Zeroize;
|
||||
|
||||
/// Reserved id of the key every probe round runs through.
|
||||
///
|
||||
/// Fixed rather than per-node or per-process: all nodes must converge on one
|
||||
/// key instead of littering the backend with probe keys, and the reserved
|
||||
/// `rustfs-internal-` prefix marks it as RustFS-owned for operators reviewing
|
||||
/// the key list.
|
||||
pub const PROBE_KEY_ID: &str = "rustfs-internal-kms-probe";
|
||||
|
||||
/// Description stored on the probe key when this node creates it.
|
||||
const PROBE_KEY_DESCRIPTION: &str = "RustFS internal KMS health probe key; created and used only by the synthetic probe";
|
||||
|
||||
/// Default interval between probe rounds.
|
||||
pub const DEFAULT_PROBE_INTERVAL: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Floor for the configured interval. Probing faster buys no additional signal
|
||||
/// and multiplies backend load by the size of the deployment.
|
||||
pub const MIN_PROBE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Probe interval in whole seconds; `0` disables the probe entirely.
|
||||
pub const ENV_KMS_PROBE_INTERVAL_SECS: &str = "RUSTFS_KMS_PROBE_INTERVAL_SECS";
|
||||
|
||||
/// Result of the most recent probe round.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProbeResult {
|
||||
/// No round has completed yet for this service version.
|
||||
Pending,
|
||||
/// The full generate/decrypt/compare cycle succeeded.
|
||||
Success,
|
||||
/// The backend cannot host the probe key; no further rounds run and no
|
||||
/// failures are recorded.
|
||||
Unsupported,
|
||||
/// The round failed with the given classification.
|
||||
Failure(ProbeFailureKind),
|
||||
}
|
||||
|
||||
impl ProbeResult {
|
||||
/// Static metric label for this result.
|
||||
fn as_label(self) -> &'static str {
|
||||
match self {
|
||||
ProbeResult::Pending => "pending",
|
||||
ProbeResult::Success => "success",
|
||||
ProbeResult::Unsupported => "unsupported",
|
||||
ProbeResult::Failure(_) => "failure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stage of the round trip that failed, used as a static metric label.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProbeFailureKind {
|
||||
/// The dedicated probe key could not be described or created.
|
||||
KeyProvisioning,
|
||||
/// Data key generation failed.
|
||||
Generate,
|
||||
/// The freshly generated ciphertext could not be decrypted.
|
||||
Decrypt,
|
||||
/// Decryption succeeded but returned material that differs from what was
|
||||
/// generated — the backend answers without round-tripping.
|
||||
Mismatch,
|
||||
}
|
||||
|
||||
impl ProbeFailureKind {
|
||||
/// Static metric label for this failure class.
|
||||
fn as_label(self) -> &'static str {
|
||||
match self {
|
||||
ProbeFailureKind::KeyProvisioning => "key_provisioning",
|
||||
ProbeFailureKind::Generate => "generate",
|
||||
ProbeFailureKind::Decrypt => "decrypt",
|
||||
ProbeFailureKind::Mismatch => "mismatch",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot of what the probe has observed so far.
|
||||
///
|
||||
/// Consumed by readiness reporting, which owns the staleness and consecutive
|
||||
/// failure thresholds; the probe itself only records what happened.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ProbeStatus {
|
||||
/// Result of the most recent completed round.
|
||||
pub result: ProbeResult,
|
||||
/// When the most recent round completed.
|
||||
pub last_round_at: Option<Instant>,
|
||||
/// When the most recent successful round completed.
|
||||
pub last_success_at: Option<Instant>,
|
||||
/// Wall-clock timestamp of the most recent success, in Unix seconds, for
|
||||
/// display and for the exported gauge. Use [`ProbeStatus::last_success_age`]
|
||||
/// for staleness decisions: it is monotonic and immune to clock steps.
|
||||
pub last_success_unix_secs: Option<i64>,
|
||||
/// Failed rounds since the last success; reset by any non-failure result.
|
||||
pub consecutive_failures: u32,
|
||||
}
|
||||
|
||||
impl ProbeStatus {
|
||||
/// Initial snapshot published before the first round completes.
|
||||
fn pending() -> Self {
|
||||
Self {
|
||||
result: ProbeResult::Pending,
|
||||
last_round_at: None,
|
||||
last_success_at: None,
|
||||
last_success_unix_secs: None,
|
||||
consecutive_failures: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// How long ago the most recent round completed.
|
||||
pub fn last_round_age(&self) -> Option<Duration> {
|
||||
self.last_round_at.map(|at| at.elapsed())
|
||||
}
|
||||
|
||||
/// How long ago the most recent successful round completed.
|
||||
pub fn last_success_age(&self) -> Option<Duration> {
|
||||
self.last_success_at.map(|at| at.elapsed())
|
||||
}
|
||||
}
|
||||
|
||||
/// Lock-free publication slot for [`ProbeStatus`].
|
||||
///
|
||||
/// Written by the single worker task and read by anyone; readers never block
|
||||
/// the probe and the probe never blocks a readiness handler.
|
||||
struct ProbeState {
|
||||
status: ArcSwap<ProbeStatus>,
|
||||
}
|
||||
|
||||
impl ProbeState {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
status: ArcSwap::from_pointee(ProbeStatus::pending()),
|
||||
}
|
||||
}
|
||||
|
||||
fn status(&self) -> Arc<ProbeStatus> {
|
||||
self.status.load_full()
|
||||
}
|
||||
|
||||
/// Fold one round result into the published snapshot.
|
||||
///
|
||||
/// Safe as a plain load-then-store because the owning worker task is the
|
||||
/// only writer.
|
||||
fn publish(&self, result: ProbeResult) {
|
||||
let previous = self.status.load();
|
||||
let now = Instant::now();
|
||||
let (last_success_at, last_success_unix_secs) = match result {
|
||||
ProbeResult::Success => (Some(now), Some(jiff::Timestamp::now().as_second())),
|
||||
_ => (previous.last_success_at, previous.last_success_unix_secs),
|
||||
};
|
||||
let consecutive_failures = match result {
|
||||
ProbeResult::Failure(_) => previous.consecutive_failures.saturating_add(1),
|
||||
_ => 0,
|
||||
};
|
||||
self.status.store(Arc::new(ProbeStatus {
|
||||
result,
|
||||
last_round_at: Some(now),
|
||||
last_success_at,
|
||||
last_success_unix_secs,
|
||||
consecutive_failures,
|
||||
}));
|
||||
record_state(result, last_success_unix_secs, consecutive_failures);
|
||||
}
|
||||
}
|
||||
|
||||
/// Owner of one service version's probe worker.
|
||||
///
|
||||
/// Mirrors the deletion worker's ownership model: cancelling stops the loop,
|
||||
/// and dropping the handle — a service version replaced without an explicit
|
||||
/// stop — cancels as a safety net.
|
||||
pub struct ProbeHandle {
|
||||
state: Arc<ProbeState>,
|
||||
cancel: CancellationToken,
|
||||
task: std::sync::Mutex<Option<tokio::task::JoinHandle<()>>>,
|
||||
}
|
||||
|
||||
impl ProbeHandle {
|
||||
/// Latest published snapshot.
|
||||
pub fn status(&self) -> Arc<ProbeStatus> {
|
||||
self.state.status()
|
||||
}
|
||||
|
||||
/// Stop the worker. The task observes the cancelled token on its next poll.
|
||||
pub(crate) fn shutdown(&self) {
|
||||
self.cancel.cancel();
|
||||
if let Ok(mut task) = self.task.lock() {
|
||||
drop(task.take());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn is_cancelled(&self) -> bool {
|
||||
self.cancel.is_cancelled()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ProbeHandle {
|
||||
fn drop(&mut self) {
|
||||
self.cancel.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the probe for a service version.
|
||||
///
|
||||
/// Returns `None` when the backend advertises no data key round trip, or when
|
||||
/// the operator disabled the probe; both cases leave no background task behind.
|
||||
pub(crate) fn spawn_probe_worker(backend: Arc<dyn KmsBackend>) -> Option<ProbeHandle> {
|
||||
let capabilities = backend.capabilities();
|
||||
if !capabilities.generate_data_key || !capabilities.decrypt {
|
||||
return None;
|
||||
}
|
||||
let interval = configured_interval()?;
|
||||
let state = Arc::new(ProbeState::new());
|
||||
let cancel = CancellationToken::new();
|
||||
let task = ProbeWorker::new(backend, state.clone(), interval).spawn(cancel.clone());
|
||||
Some(ProbeHandle {
|
||||
state,
|
||||
cancel,
|
||||
task: std::sync::Mutex::new(Some(task)),
|
||||
})
|
||||
}
|
||||
|
||||
/// Probe interval from the environment, or `None` when the probe is disabled.
|
||||
fn configured_interval() -> Option<Duration> {
|
||||
parse_interval(std::env::var(ENV_KMS_PROBE_INTERVAL_SECS).ok().as_deref())
|
||||
}
|
||||
|
||||
/// Interpret the configured interval: unset or unparsable falls back to the
|
||||
/// default, `0` disables the probe, and anything below [`MIN_PROBE_INTERVAL`]
|
||||
/// is raised to it so a misconfigured value cannot turn the probe into a load
|
||||
/// generator against the backend.
|
||||
fn parse_interval(value: Option<&str>) -> Option<Duration> {
|
||||
let Some(value) = value else {
|
||||
return Some(DEFAULT_PROBE_INTERVAL);
|
||||
};
|
||||
let Ok(seconds) = value.trim().parse::<u64>() else {
|
||||
warn!(
|
||||
variable = ENV_KMS_PROBE_INTERVAL_SECS,
|
||||
"ignoring unparsable KMS probe interval; falling back to the default"
|
||||
);
|
||||
return Some(DEFAULT_PROBE_INTERVAL);
|
||||
};
|
||||
if seconds == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(Duration::from_secs(seconds).max(MIN_PROBE_INTERVAL))
|
||||
}
|
||||
|
||||
/// Encryption context bound to every probe data key.
|
||||
///
|
||||
/// Static and free of per-node values, so material generated by one node stays
|
||||
/// decryptable by every other node running the same probe.
|
||||
fn probe_encryption_context() -> HashMap<String, String> {
|
||||
HashMap::from([("rustfs-purpose".to_string(), "kms-probe".to_string())])
|
||||
}
|
||||
|
||||
/// Outcome of making sure the dedicated probe key exists.
|
||||
enum KeyProvisioning {
|
||||
Ready,
|
||||
Unsupported,
|
||||
Failed,
|
||||
}
|
||||
|
||||
pub(crate) struct ProbeWorker {
|
||||
backend: Arc<dyn KmsBackend>,
|
||||
state: Arc<ProbeState>,
|
||||
interval: Duration,
|
||||
}
|
||||
|
||||
impl ProbeWorker {
|
||||
fn new(backend: Arc<dyn KmsBackend>, state: Arc<ProbeState>, interval: Duration) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
state,
|
||||
interval,
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn(self, cancel: CancellationToken) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move { self.run(cancel).await })
|
||||
}
|
||||
|
||||
async fn run(self, cancel: CancellationToken) {
|
||||
describe_metrics();
|
||||
let mut ticker = tokio::time::interval(self.interval);
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {
|
||||
debug!("KMS probe worker stopped");
|
||||
return;
|
||||
}
|
||||
_ = ticker.tick() => {}
|
||||
}
|
||||
let started = Instant::now();
|
||||
let result = self.round().await;
|
||||
record_round(result, started.elapsed());
|
||||
self.state.publish(result);
|
||||
if result == ProbeResult::Unsupported {
|
||||
// A capability gap does not change while a service version
|
||||
// lives, so retrying it forever would only produce noise.
|
||||
info!("KMS backend cannot host the synthetic probe key; probe disabled for this service version");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one round. Exposed separately so tests can drive the round trip
|
||||
/// deterministically without the interval loop.
|
||||
pub(crate) async fn round(&self) -> ProbeResult {
|
||||
let capabilities = self.backend.capabilities();
|
||||
if !capabilities.generate_data_key || !capabilities.decrypt {
|
||||
return ProbeResult::Unsupported;
|
||||
}
|
||||
match self.ensure_probe_key().await {
|
||||
KeyProvisioning::Ready => {}
|
||||
KeyProvisioning::Unsupported => return ProbeResult::Unsupported,
|
||||
KeyProvisioning::Failed => return ProbeResult::Failure(ProbeFailureKind::KeyProvisioning),
|
||||
}
|
||||
|
||||
let mut generated = match self
|
||||
.backend
|
||||
.generate_data_key(GenerateDataKeyRequest {
|
||||
key_id: PROBE_KEY_ID.to_string(),
|
||||
key_spec: KeySpec::Aes256,
|
||||
encryption_context: probe_encryption_context(),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(generated) => generated,
|
||||
Err(error) => {
|
||||
warn!(%error, "KMS probe could not generate a data key");
|
||||
return ProbeResult::Failure(ProbeFailureKind::Generate);
|
||||
}
|
||||
};
|
||||
|
||||
let decrypted = self
|
||||
.backend
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: generated.ciphertext_blob.clone(),
|
||||
encryption_context: probe_encryption_context(),
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
.await;
|
||||
|
||||
let result = match decrypted {
|
||||
Ok(mut response) => {
|
||||
let round_trips = bool::from(response.plaintext.ct_eq(&generated.plaintext_key));
|
||||
response.plaintext.zeroize();
|
||||
if round_trips {
|
||||
debug!("KMS probe round trip succeeded");
|
||||
ProbeResult::Success
|
||||
} else {
|
||||
// The backend answered both calls but the material does not
|
||||
// survive the round trip: report it as loudly as an outage.
|
||||
warn!("KMS probe decrypted material does not match the generated data key");
|
||||
ProbeResult::Failure(ProbeFailureKind::Mismatch)
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
warn!(%error, "KMS probe could not decrypt its generated data key");
|
||||
ProbeResult::Failure(ProbeFailureKind::Decrypt)
|
||||
}
|
||||
};
|
||||
generated.plaintext_key.zeroize();
|
||||
result
|
||||
}
|
||||
|
||||
/// Create the probe key if it is missing.
|
||||
///
|
||||
/// Idempotent by construction: an existing key is used as is, and a create
|
||||
/// that loses the race against another node reports the key as ready
|
||||
/// instead of failing the round.
|
||||
async fn ensure_probe_key(&self) -> KeyProvisioning {
|
||||
match self
|
||||
.backend
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: PROBE_KEY_ID.to_string(),
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => return KeyProvisioning::Ready,
|
||||
Err(KmsError::KeyNotFound { .. }) => {}
|
||||
Err(error) if is_capability_gap(&error) => return KeyProvisioning::Unsupported,
|
||||
Err(error) => {
|
||||
warn!(%error, "KMS probe could not describe its probe key");
|
||||
return KeyProvisioning::Failed;
|
||||
}
|
||||
}
|
||||
|
||||
match self
|
||||
.backend
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some(PROBE_KEY_ID.to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some(PROBE_KEY_DESCRIPTION.to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
info!(key_id = PROBE_KEY_ID, "created the KMS synthetic probe key");
|
||||
KeyProvisioning::Ready
|
||||
}
|
||||
// Another node created the key between the describe and the create.
|
||||
Err(KmsError::KeyAlreadyExists { .. }) => KeyProvisioning::Ready,
|
||||
Err(error) if is_capability_gap(&error) => KeyProvisioning::Unsupported,
|
||||
Err(error) => {
|
||||
warn!(%error, "KMS probe could not create its probe key");
|
||||
KeyProvisioning::Failed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an error means "this backend will never host a probe key".
|
||||
///
|
||||
/// Read-only single-key backends reject key creation outright, and backends
|
||||
/// without the operation report an unsupported capability. Both are properties
|
||||
/// of the configuration rather than symptoms of an outage, so they stop the
|
||||
/// probe instead of counting as failures.
|
||||
fn is_capability_gap(error: &KmsError) -> bool {
|
||||
matches!(error, KmsError::UnsupportedCapability { .. } | KmsError::InvalidOperation { .. })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Metrics
|
||||
//
|
||||
// Label values are exclusively static outcome classes; the probe key id is a
|
||||
// compile-time constant and still never becomes a label, so the metric shape
|
||||
// stays independent of how many keys or nodes exist.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Counter: probe rounds completed, by `result`.
|
||||
const METRIC_PROBE_ROUNDS_TOTAL: &str = "rustfs_kms_probe_rounds_total";
|
||||
/// Counter: failed probe rounds, by `failure_kind`.
|
||||
const METRIC_PROBE_FAILURES_TOTAL: &str = "rustfs_kms_probe_failures_total";
|
||||
/// Histogram: wall-clock duration of one probe round, in seconds, by `result`.
|
||||
const METRIC_PROBE_DURATION_SECONDS: &str = "rustfs_kms_probe_duration_seconds";
|
||||
/// Gauge: Unix timestamp of the most recent successful round.
|
||||
const METRIC_PROBE_LAST_SUCCESS_TIMESTAMP: &str = "rustfs_kms_probe_last_success_timestamp_seconds";
|
||||
/// Gauge: failed rounds since the last success.
|
||||
const METRIC_PROBE_CONSECUTIVE_FAILURES: &str = "rustfs_kms_probe_consecutive_failures";
|
||||
|
||||
/// Register metric descriptions once per process.
|
||||
fn describe_metrics() {
|
||||
static DESCRIBE: std::sync::Once = std::sync::Once::new();
|
||||
DESCRIBE.call_once(|| {
|
||||
metrics::describe_counter!(METRIC_PROBE_ROUNDS_TOTAL, "Total synthetic KMS probe rounds completed, by result");
|
||||
metrics::describe_counter!(
|
||||
METRIC_PROBE_FAILURES_TOTAL,
|
||||
"Total failed synthetic KMS probe rounds, by the round-trip stage that failed"
|
||||
);
|
||||
metrics::describe_histogram!(
|
||||
METRIC_PROBE_DURATION_SECONDS,
|
||||
"Wall-clock duration of one synthetic KMS probe round, in seconds"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
METRIC_PROBE_LAST_SUCCESS_TIMESTAMP,
|
||||
"Unix timestamp of the most recent successful synthetic KMS probe round"
|
||||
);
|
||||
metrics::describe_gauge!(
|
||||
METRIC_PROBE_CONSECUTIVE_FAILURES,
|
||||
"Synthetic KMS probe rounds that have failed since the last success"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// Record one completed round.
|
||||
///
|
||||
/// An unsupported backend is deliberately counted as its own result and never
|
||||
/// as a failure, so alerts on the failure counter stay silent for deployments
|
||||
/// whose backend cannot host the probe.
|
||||
fn record_round(result: ProbeResult, elapsed: Duration) {
|
||||
metrics::counter!(METRIC_PROBE_ROUNDS_TOTAL, "result" => result.as_label()).increment(1);
|
||||
if let ProbeResult::Failure(kind) = result {
|
||||
metrics::counter!(METRIC_PROBE_FAILURES_TOTAL, "failure_kind" => kind.as_label()).increment(1);
|
||||
}
|
||||
metrics::histogram!(METRIC_PROBE_DURATION_SECONDS, "result" => result.as_label()).record(elapsed.as_secs_f64());
|
||||
}
|
||||
|
||||
/// Mirror the published snapshot into the gauges.
|
||||
fn record_state(result: ProbeResult, last_success_unix_secs: Option<i64>, consecutive_failures: u32) {
|
||||
if result == ProbeResult::Success
|
||||
&& let Some(timestamp) = last_success_unix_secs
|
||||
{
|
||||
// Only ever moved forward by a success, so a failing probe leaves the
|
||||
// gauge at the last known good time and its age keeps growing.
|
||||
metrics::gauge!(METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).set(timestamp as f64);
|
||||
}
|
||||
metrics::gauge!(METRIC_PROBE_CONSECUTIVE_FAILURES).set(f64::from(consecutive_failures));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::backends::local::LocalKmsBackend;
|
||||
use crate::backends::static_kms::StaticKmsBackend;
|
||||
use crate::config::KmsConfig;
|
||||
use crate::types::{
|
||||
CancelKeyDeletionRequest, CancelKeyDeletionResponse, CreateKeyResponse, DecryptResponse, DeleteKeyRequest,
|
||||
DeleteKeyResponse, DescribeKeyResponse, EncryptRequest, EncryptResponse, GenerateDataKeyResponse, KeyMetadata, KeyState,
|
||||
ListKeysRequest, ListKeysResponse,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use base64::Engine as _;
|
||||
use metrics_util::MetricKind;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use std::future::Future;
|
||||
|
||||
async fn local_backend(temp_dir: &tempfile::TempDir) -> Arc<dyn KmsBackend> {
|
||||
let config = KmsConfig::local(temp_dir.path().to_path_buf()).with_insecure_development_defaults();
|
||||
Arc::new(LocalKmsBackend::new(config).await.expect("local backend should build"))
|
||||
}
|
||||
|
||||
async fn static_backend() -> Arc<dyn KmsBackend> {
|
||||
let config =
|
||||
KmsConfig::static_kms("static-key".to_string(), base64::engine::general_purpose::STANDARD.encode([0x42u8; 32]));
|
||||
Arc::new(StaticKmsBackend::new(config).await.expect("static backend should build"))
|
||||
}
|
||||
|
||||
fn worker(backend: Arc<dyn KmsBackend>) -> ProbeWorker {
|
||||
ProbeWorker::new(backend, Arc::new(ProbeState::new()), DEFAULT_PROBE_INTERVAL)
|
||||
}
|
||||
|
||||
/// Backend that answers every probe call but corrupts the round trip in a
|
||||
/// configurable way, so each failure classification can be provoked without
|
||||
/// a real backend outage.
|
||||
struct BrokenBackend {
|
||||
fault: Fault,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum Fault {
|
||||
Generate,
|
||||
Decrypt,
|
||||
/// Decrypt succeeds but returns material other than the generated key.
|
||||
Mismatch,
|
||||
}
|
||||
|
||||
impl BrokenBackend {
|
||||
fn arc(fault: Fault) -> Arc<dyn KmsBackend> {
|
||||
Arc::new(Self { fault })
|
||||
}
|
||||
|
||||
fn probe_key_metadata() -> KeyMetadata {
|
||||
KeyMetadata {
|
||||
key_id: PROBE_KEY_ID.to_string(),
|
||||
key_state: KeyState::Enabled,
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: None,
|
||||
creation_date: jiff::Zoned::now(),
|
||||
deletion_date: None,
|
||||
origin: "test".to_string(),
|
||||
key_manager: "test".to_string(),
|
||||
tags: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl KmsBackend for BrokenBackend {
|
||||
async fn create_key(&self, _request: CreateKeyRequest) -> crate::error::Result<CreateKeyResponse> {
|
||||
unimplemented!("the probe key always exists on this backend")
|
||||
}
|
||||
|
||||
async fn encrypt(&self, _request: EncryptRequest) -> crate::error::Result<EncryptResponse> {
|
||||
unimplemented!("not exercised by probe tests")
|
||||
}
|
||||
|
||||
async fn decrypt(&self, _request: DecryptRequest) -> crate::error::Result<DecryptResponse> {
|
||||
match self.fault {
|
||||
Fault::Decrypt => Err(KmsError::backend_error("decrypt unavailable")),
|
||||
Fault::Mismatch => Ok(DecryptResponse {
|
||||
plaintext: vec![0xffu8; 32],
|
||||
key_id: PROBE_KEY_ID.to_string(),
|
||||
encryption_algorithm: None,
|
||||
}),
|
||||
Fault::Generate => unreachable!("generation fails before decrypt is reached"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn generate_data_key(&self, _request: GenerateDataKeyRequest) -> crate::error::Result<GenerateDataKeyResponse> {
|
||||
match self.fault {
|
||||
Fault::Generate => Err(KmsError::backend_error("data key generation unavailable")),
|
||||
_ => Ok(GenerateDataKeyResponse {
|
||||
key_id: PROBE_KEY_ID.to_string(),
|
||||
plaintext_key: vec![0x11u8; 32],
|
||||
ciphertext_blob: vec![0x22u8; 48],
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn describe_key(&self, _request: DescribeKeyRequest) -> crate::error::Result<DescribeKeyResponse> {
|
||||
Ok(DescribeKeyResponse {
|
||||
key_metadata: Self::probe_key_metadata(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_keys(&self, _request: ListKeysRequest) -> crate::error::Result<ListKeysResponse> {
|
||||
unimplemented!("not exercised by probe tests")
|
||||
}
|
||||
|
||||
async fn delete_key(&self, _request: DeleteKeyRequest) -> crate::error::Result<DeleteKeyResponse> {
|
||||
unimplemented!("not exercised by probe tests")
|
||||
}
|
||||
|
||||
async fn cancel_key_deletion(
|
||||
&self,
|
||||
_request: CancelKeyDeletionRequest,
|
||||
) -> crate::error::Result<CancelKeyDeletionResponse> {
|
||||
unimplemented!("not exercised by probe tests")
|
||||
}
|
||||
|
||||
async fn health_check(&self) -> crate::error::Result<bool> {
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn round_creates_the_probe_key_once_and_round_trips_it() {
|
||||
let temp_dir = tempfile::tempdir().expect("temp dir");
|
||||
let backend = local_backend(&temp_dir).await;
|
||||
let worker = worker(backend.clone());
|
||||
|
||||
assert_eq!(worker.round().await, ProbeResult::Success);
|
||||
let created = backend
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: PROBE_KEY_ID.to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("the first round must create the probe key");
|
||||
assert_eq!(created.key_metadata.key_id, PROBE_KEY_ID);
|
||||
|
||||
// A second round reuses the key instead of failing on the existing one,
|
||||
// which is also what a second node in the deployment would do.
|
||||
assert_eq!(worker.round().await, ProbeResult::Success);
|
||||
assert_eq!(
|
||||
backend
|
||||
.list_keys(ListKeysRequest {
|
||||
limit: Some(100),
|
||||
marker: None,
|
||||
usage_filter: None,
|
||||
status_filter: None,
|
||||
})
|
||||
.await
|
||||
.expect("list keys")
|
||||
.keys
|
||||
.iter()
|
||||
.filter(|key| key.key_id == PROBE_KEY_ID)
|
||||
.count(),
|
||||
1,
|
||||
"probe key creation must be idempotent"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_only_backend_is_reported_unsupported_without_failures() {
|
||||
let (snapshot, result) = record_metrics(|| {
|
||||
Box::pin(async {
|
||||
let worker = worker(static_backend().await);
|
||||
let result = worker.round().await;
|
||||
record_round(result, Duration::from_millis(1));
|
||||
result
|
||||
})
|
||||
});
|
||||
|
||||
assert_eq!(result, ProbeResult::Unsupported);
|
||||
assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "unsupported")]), 1);
|
||||
assert_eq!(
|
||||
counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[]),
|
||||
0,
|
||||
"an unsupported backend must never raise probe failures"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn each_round_trip_stage_gets_its_own_failure_classification() {
|
||||
for (fault, expected) in [
|
||||
(Fault::Generate, ProbeFailureKind::Generate),
|
||||
(Fault::Decrypt, ProbeFailureKind::Decrypt),
|
||||
(Fault::Mismatch, ProbeFailureKind::Mismatch),
|
||||
] {
|
||||
assert_eq!(worker(BrokenBackend::arc(fault)).round().await, ProbeResult::Failure(expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metrics_and_status_record_success_then_failure() {
|
||||
let (snapshot, status) = record_metrics(|| {
|
||||
Box::pin(async {
|
||||
let temp_dir = tempfile::tempdir().expect("temp dir");
|
||||
let state = Arc::new(ProbeState::new());
|
||||
let healthy = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL);
|
||||
let round = healthy.round().await;
|
||||
record_round(round, Duration::from_millis(10));
|
||||
state.publish(round);
|
||||
|
||||
let broken = ProbeWorker::new(BrokenBackend::arc(Fault::Mismatch), state.clone(), DEFAULT_PROBE_INTERVAL);
|
||||
for _ in 0..2 {
|
||||
let round = broken.round().await;
|
||||
record_round(round, Duration::from_millis(10));
|
||||
state.publish(round);
|
||||
}
|
||||
state.status()
|
||||
})
|
||||
});
|
||||
|
||||
assert_eq!(status.result, ProbeResult::Failure(ProbeFailureKind::Mismatch));
|
||||
assert_eq!(status.consecutive_failures, 2);
|
||||
assert!(
|
||||
status.last_success_at.is_some() && status.last_success_unix_secs.is_some(),
|
||||
"a failing probe must keep reporting when it last succeeded"
|
||||
);
|
||||
|
||||
assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "success")]), 1);
|
||||
assert_eq!(counter_value(&snapshot, METRIC_PROBE_ROUNDS_TOTAL, &[("result", "failure")]), 2);
|
||||
assert_eq!(counter_value(&snapshot, METRIC_PROBE_FAILURES_TOTAL, &[("failure_kind", "mismatch")]), 2);
|
||||
assert_eq!(histogram_values(&snapshot, METRIC_PROBE_DURATION_SECONDS, &[]).len(), 3);
|
||||
assert!(
|
||||
gauge_value(&snapshot, METRIC_PROBE_LAST_SUCCESS_TIMESTAMP).is_some_and(|timestamp| timestamp > 0.0),
|
||||
"the last success gauge must carry a wall-clock timestamp"
|
||||
);
|
||||
assert_eq!(gauge_value(&snapshot, METRIC_PROBE_CONSECUTIVE_FAILURES), Some(2.0));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn worker_publishes_status_and_stops_on_cancel() {
|
||||
let temp_dir = tempfile::tempdir().expect("temp dir");
|
||||
let state = Arc::new(ProbeState::new());
|
||||
assert_eq!(state.status().result, ProbeResult::Pending);
|
||||
|
||||
let cancel = CancellationToken::new();
|
||||
let task = ProbeWorker::new(local_backend(&temp_dir).await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
|
||||
|
||||
// The paused clock auto-advances through the worker's interval ticks.
|
||||
let mut observed = None;
|
||||
for _ in 0..100 {
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
if state.status().result != ProbeResult::Pending {
|
||||
observed = Some(state.status());
|
||||
break;
|
||||
}
|
||||
}
|
||||
let observed = observed.expect("the worker loop must publish a round result");
|
||||
assert_eq!(observed.result, ProbeResult::Success);
|
||||
assert_eq!(observed.consecutive_failures, 0);
|
||||
assert!(observed.last_success_at.is_some());
|
||||
|
||||
cancel.cancel();
|
||||
task.await.expect("worker task must stop after cancellation");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unsupported_backend_stops_the_worker_without_cancellation() {
|
||||
let state = Arc::new(ProbeState::new());
|
||||
let cancel = CancellationToken::new();
|
||||
let task = ProbeWorker::new(static_backend().await, state.clone(), DEFAULT_PROBE_INTERVAL).spawn(cancel.clone());
|
||||
|
||||
task.await.expect("worker must exit on its own for an unsupported backend");
|
||||
assert_eq!(state.status().result, ProbeResult::Unsupported);
|
||||
assert!(!cancel.is_cancelled());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interval_configuration_defaults_clamps_and_disables() {
|
||||
assert_eq!(parse_interval(None), Some(DEFAULT_PROBE_INTERVAL));
|
||||
assert_eq!(parse_interval(Some("not-a-number")), Some(DEFAULT_PROBE_INTERVAL));
|
||||
assert_eq!(parse_interval(Some(" 120 ")), Some(Duration::from_secs(120)));
|
||||
assert_eq!(parse_interval(Some("1")), Some(MIN_PROBE_INTERVAL));
|
||||
assert_eq!(parse_interval(Some("0")), None, "zero must disable the probe");
|
||||
}
|
||||
|
||||
// -- Metric assertion helpers, mirroring the policy engine's tests --------
|
||||
|
||||
type MetricEntry = (
|
||||
metrics_util::CompositeKey,
|
||||
Option<metrics::Unit>,
|
||||
Option<metrics::SharedString>,
|
||||
DebugValue,
|
||||
);
|
||||
|
||||
/// Run `test` on a paused current-thread runtime under a debugging recorder
|
||||
/// and return one snapshot of everything it emitted.
|
||||
fn record_metrics<Out>(test: impl FnOnce() -> std::pin::Pin<Box<dyn Future<Output = Out>>>) -> (Vec<MetricEntry>, Out) {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let out = metrics::with_local_recorder(&recorder, || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_time()
|
||||
.start_paused(true)
|
||||
.build()
|
||||
.expect("current-thread runtime must build");
|
||||
runtime.block_on(test())
|
||||
});
|
||||
(snapshotter.snapshot().into_vec(), out)
|
||||
}
|
||||
|
||||
fn labels_match(key: &metrics::Key, labels: &[(&str, &str)]) -> bool {
|
||||
labels
|
||||
.iter()
|
||||
.all(|(label, expected)| key.labels().any(|l| l.key() == *label && l.value() == *expected))
|
||||
}
|
||||
|
||||
fn counter_value(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> u64 {
|
||||
snapshot
|
||||
.iter()
|
||||
.filter_map(|(composite, _unit, _description, value)| {
|
||||
let matches = composite.kind() == MetricKind::Counter
|
||||
&& composite.key().name() == name
|
||||
&& labels_match(composite.key(), labels);
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Counter(count)) => Some(*count),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.sum()
|
||||
}
|
||||
|
||||
fn histogram_values(snapshot: &[MetricEntry], name: &str, labels: &[(&str, &str)]) -> Vec<f64> {
|
||||
snapshot
|
||||
.iter()
|
||||
.filter_map(|(composite, _unit, _description, value)| {
|
||||
let matches = composite.kind() == MetricKind::Histogram
|
||||
&& composite.key().name() == name
|
||||
&& labels_match(composite.key(), labels);
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Histogram(values)) => Some(values),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.flatten()
|
||||
.map(|value| value.into_inner())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn gauge_value(snapshot: &[MetricEntry], name: &str) -> Option<f64> {
|
||||
snapshot.iter().find_map(|(composite, _unit, _description, value)| {
|
||||
let matches = composite.kind() == MetricKind::Gauge && composite.key().name() == name;
|
||||
match (matches, value) {
|
||||
(true, DebugValue::Gauge(gauge)) => Some(gauge.into_inner()),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+2
-233
@@ -14,10 +14,6 @@
|
||||
|
||||
//! Object encryption service for S3-compatible encryption
|
||||
|
||||
use crate::api_types::{
|
||||
TagKeyRequest, TagKeyResponse, UntagKeyRequest, UntagKeyResponse, UpdateKeyDescriptionRequest, UpdateKeyDescriptionResponse,
|
||||
};
|
||||
use crate::cache::KmsCacheStats;
|
||||
use crate::encryption::ciphers::{create_cipher, generate_iv};
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
@@ -116,23 +112,6 @@ impl ObjectEncryptionService {
|
||||
self.kms_manager.create_key(request).await
|
||||
}
|
||||
|
||||
/// Create a new master key on behalf of `context`'s principal
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - CreateKeyRequest with key parameters
|
||||
/// * `context` - Identity and correlation data recorded in the audit trail
|
||||
///
|
||||
/// # Returns
|
||||
/// CreateKeyResponse with created key details
|
||||
///
|
||||
pub async fn create_key_with_context(
|
||||
&self,
|
||||
request: CreateKeyRequest,
|
||||
context: &OperationContext,
|
||||
) -> Result<CreateKeyResponse> {
|
||||
self.kms_manager.create_key_with_context(request, context).await
|
||||
}
|
||||
|
||||
/// Describe a master key (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -145,23 +124,6 @@ impl ObjectEncryptionService {
|
||||
self.kms_manager.describe_key(request).await
|
||||
}
|
||||
|
||||
/// Describe a master key on behalf of `context`'s principal
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - DescribeKeyRequest with key ID
|
||||
/// * `context` - Identity and correlation data recorded in the audit trail
|
||||
///
|
||||
/// # Returns
|
||||
/// DescribeKeyResponse with key metadata
|
||||
///
|
||||
pub async fn describe_key_with_context(
|
||||
&self,
|
||||
request: DescribeKeyRequest,
|
||||
context: &OperationContext,
|
||||
) -> Result<DescribeKeyResponse> {
|
||||
self.kms_manager.describe_key_with_context(request, context).await
|
||||
}
|
||||
|
||||
/// List master keys (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -174,77 +136,6 @@ impl ObjectEncryptionService {
|
||||
self.kms_manager.list_keys(request).await
|
||||
}
|
||||
|
||||
/// List master keys on behalf of `context`'s principal
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - ListKeysRequest with listing parameters
|
||||
/// * `context` - Identity and correlation data recorded in the audit trail
|
||||
///
|
||||
/// # Returns
|
||||
/// ListKeysResponse with list of keys
|
||||
///
|
||||
pub async fn list_keys_with_context(&self, request: ListKeysRequest, context: &OperationContext) -> Result<ListKeysResponse> {
|
||||
self.kms_manager.list_keys_with_context(request, context).await
|
||||
}
|
||||
|
||||
/// Replace a master key's description (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - UpdateKeyDescriptionRequest with key ID and the new
|
||||
/// description; an empty description clears the stored value
|
||||
///
|
||||
/// # Returns
|
||||
/// UpdateKeyDescriptionResponse acknowledging the update
|
||||
///
|
||||
pub async fn update_key_description(&self, request: UpdateKeyDescriptionRequest) -> Result<UpdateKeyDescriptionResponse> {
|
||||
let description = (!request.description.is_empty()).then_some(request.description.as_str());
|
||||
self.kms_manager.update_key_description(&request.key_id, description).await?;
|
||||
|
||||
Ok(UpdateKeyDescriptionResponse {
|
||||
success: true,
|
||||
message: "key description updated".to_string(),
|
||||
key_id: request.key_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Add or overwrite master key tags (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - TagKeyRequest with key ID and the tags to set; tags
|
||||
/// outside the request are left untouched
|
||||
///
|
||||
/// # Returns
|
||||
/// TagKeyResponse acknowledging the update
|
||||
///
|
||||
pub async fn tag_key(&self, request: TagKeyRequest) -> Result<TagKeyResponse> {
|
||||
self.kms_manager.tag_key(&request.key_id, &request.tags).await?;
|
||||
|
||||
Ok(TagKeyResponse {
|
||||
success: true,
|
||||
message: "key tags updated".to_string(),
|
||||
key_id: request.key_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Remove master key tags (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `request` - UntagKeyRequest with key ID and the tag keys to remove;
|
||||
/// tag keys that are not set are ignored, so the call is idempotent
|
||||
///
|
||||
/// # Returns
|
||||
/// UntagKeyResponse acknowledging the removal
|
||||
///
|
||||
pub async fn untag_key(&self, request: UntagKeyRequest) -> Result<UntagKeyResponse> {
|
||||
self.kms_manager.untag_key(&request.key_id, &request.tag_keys).await?;
|
||||
|
||||
Ok(UntagKeyResponse {
|
||||
success: true,
|
||||
message: "key tags removed".to_string(),
|
||||
key_id: request.key_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Generate a data encryption key (delegates to KMS manager)
|
||||
///
|
||||
/// # Arguments
|
||||
@@ -269,9 +160,9 @@ impl ObjectEncryptionService {
|
||||
/// Get cache statistics
|
||||
///
|
||||
/// # Returns
|
||||
/// A [`KmsCacheStats`] snapshot if caching is enabled, `None` otherwise
|
||||
/// Option with (hits, misses) if caching is enabled
|
||||
///
|
||||
pub async fn cache_stats(&self) -> Option<KmsCacheStats> {
|
||||
pub async fn cache_stats(&self) -> Option<(u64, u64)> {
|
||||
self.kms_manager.cache_stats().await
|
||||
}
|
||||
|
||||
@@ -1027,128 +918,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn describe(service: &ObjectEncryptionService, key_id: &str) -> KeyMetadata {
|
||||
service
|
||||
.describe_key(DescribeKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("describe should succeed")
|
||||
.key_metadata
|
||||
}
|
||||
|
||||
async fn create_metadata_test_key(service: &ObjectEncryptionService, key_id: &str) {
|
||||
service
|
||||
.create_key(CreateKeyRequest {
|
||||
key_name: Some(key_id.to_string()),
|
||||
key_usage: KeyUsage::EncryptDecrypt,
|
||||
description: Some("original".to_string()),
|
||||
policy: None,
|
||||
tags: HashMap::from([
|
||||
("name".to_string(), key_id.to_string()),
|
||||
("team".to_string(), "storage".to_string()),
|
||||
]),
|
||||
origin: None,
|
||||
})
|
||||
.await
|
||||
.expect("test key should be created");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn key_metadata_updates_are_visible_to_describe() {
|
||||
let (service, _temp_dir) = create_test_service().await;
|
||||
create_metadata_test_key(&service, "metadata-key").await;
|
||||
|
||||
service
|
||||
.update_key_description(UpdateKeyDescriptionRequest {
|
||||
key_id: "metadata-key".to_string(),
|
||||
description: "updated".to_string(),
|
||||
})
|
||||
.await
|
||||
.expect("description update should succeed");
|
||||
service
|
||||
.tag_key(TagKeyRequest {
|
||||
key_id: "metadata-key".to_string(),
|
||||
tags: HashMap::from([
|
||||
("team".to_string(), "platform".to_string()),
|
||||
("env".to_string(), "prod".to_string()),
|
||||
]),
|
||||
})
|
||||
.await
|
||||
.expect("tagging should succeed");
|
||||
|
||||
// Reading through the manager's metadata cache must observe the
|
||||
// updates, not the snapshot cached when the key was created.
|
||||
let metadata = describe(&service, "metadata-key").await;
|
||||
assert_eq!(metadata.description.as_deref(), Some("updated"));
|
||||
assert_eq!(metadata.tags.get("team").map(String::as_str), Some("platform"));
|
||||
assert_eq!(metadata.tags.get("env").map(String::as_str), Some("prod"));
|
||||
assert_eq!(
|
||||
metadata.tags.get("name").map(String::as_str),
|
||||
Some("metadata-key"),
|
||||
"tags set at creation must survive a later tag update"
|
||||
);
|
||||
|
||||
// Untagging is idempotent: removing an absent tag is a no-op, not an
|
||||
// error, so a retried request stays safe.
|
||||
for attempt in 1..=2 {
|
||||
service
|
||||
.untag_key(UntagKeyRequest {
|
||||
key_id: "metadata-key".to_string(),
|
||||
tag_keys: vec!["env".to_string()],
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("untag attempt {attempt} should succeed: {error:?}"));
|
||||
}
|
||||
let metadata = describe(&service, "metadata-key").await;
|
||||
assert!(!metadata.tags.contains_key("env"), "untagged tag must be gone: {:?}", metadata.tags);
|
||||
assert_eq!(
|
||||
metadata.tags.get("team").map(String::as_str),
|
||||
Some("platform"),
|
||||
"untagging must not touch other tags"
|
||||
);
|
||||
|
||||
// An empty description clears the stored value.
|
||||
service
|
||||
.update_key_description(UpdateKeyDescriptionRequest {
|
||||
key_id: "metadata-key".to_string(),
|
||||
description: String::new(),
|
||||
})
|
||||
.await
|
||||
.expect("clearing the description should succeed");
|
||||
assert_eq!(describe(&service, "metadata-key").await.description, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn identity_tag_is_not_writable_through_metadata_updates() {
|
||||
let (service, _temp_dir) = create_test_service().await;
|
||||
create_metadata_test_key(&service, "identity-key").await;
|
||||
|
||||
let rewrite = service
|
||||
.tag_key(TagKeyRequest {
|
||||
key_id: "identity-key".to_string(),
|
||||
tags: HashMap::from([("name".to_string(), "other-key".to_string())]),
|
||||
})
|
||||
.await
|
||||
.expect_err("rewriting the identity tag must be rejected");
|
||||
assert!(matches!(rewrite, KmsError::InvalidOperation { .. }), "got {rewrite:?}");
|
||||
|
||||
let removal = service
|
||||
.untag_key(UntagKeyRequest {
|
||||
key_id: "identity-key".to_string(),
|
||||
tag_keys: vec!["team".to_string(), "name".to_string()],
|
||||
})
|
||||
.await
|
||||
.expect_err("removing the identity tag must be rejected");
|
||||
assert!(matches!(removal, KmsError::InvalidOperation { .. }), "got {removal:?}");
|
||||
|
||||
// Both rejections are pre-write: the record is untouched, including
|
||||
// the ordinary tag that shared the rejected untag request.
|
||||
let metadata = describe(&service, "identity-key").await;
|
||||
assert_eq!(metadata.tags.get("name").map(String::as_str), Some("identity-key"));
|
||||
assert_eq!(metadata.tags.get("team").map(String::as_str), Some("storage"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_decrypt_data_key_uses_object_encryption_context() {
|
||||
let (service, _temp_dir) = create_test_service().await;
|
||||
|
||||
@@ -14,14 +14,12 @@
|
||||
|
||||
//! KMS service manager for dynamic configuration and runtime management
|
||||
|
||||
use crate::audit::KmsAuditSink;
|
||||
use crate::backends::vault_credentials::CredentialTaskHandle;
|
||||
use crate::backends::{KmsBackend, local::LocalKmsBackend};
|
||||
use crate::config::{BackendConfig, KmsConfig};
|
||||
use crate::deletion_worker::{DeletionReferenceChecker, DeletionWorker};
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::manager::KmsManager;
|
||||
use crate::probe::{ProbeHandle, ProbeStatus, spawn_probe_worker};
|
||||
use crate::service::ObjectEncryptionService;
|
||||
use arc_swap::ArcSwap;
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -122,23 +120,13 @@ struct ServiceVersion {
|
||||
/// Background deletion worker owned by this service version, if the
|
||||
/// backend supports deletion scheduling
|
||||
deletion_worker: Option<Arc<DeletionWorkerHandle>>,
|
||||
/// Background synthetic probe owned by this service version, if the
|
||||
/// backend can round-trip a data key and the probe is enabled
|
||||
probe_worker: Option<Arc<ProbeHandle>>,
|
||||
}
|
||||
|
||||
impl ServiceVersion {
|
||||
/// Stop the background workers this version owns.
|
||||
///
|
||||
/// The credential renewal task is recycled separately because its shutdown
|
||||
/// is asynchronous.
|
||||
fn shutdown_background_workers(&self) {
|
||||
fn shutdown_deletion_worker(&self) {
|
||||
if let Some(worker) = &self.deletion_worker {
|
||||
worker.shutdown();
|
||||
}
|
||||
if let Some(probe) = &self.probe_worker {
|
||||
probe.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,8 +171,6 @@ pub struct KmsServiceManager {
|
||||
lifecycle_mutex: Arc<Mutex<()>>,
|
||||
/// External reference checker consulted before expired keys are removed
|
||||
deletion_reference_checker: std::sync::RwLock<Option<Arc<dyn DeletionReferenceChecker>>>,
|
||||
/// External sink receiving audit records for management operations
|
||||
audit_sink: std::sync::RwLock<Option<Arc<dyn KmsAuditSink>>>,
|
||||
}
|
||||
|
||||
impl KmsServiceManager {
|
||||
@@ -199,26 +185,9 @@ impl KmsServiceManager {
|
||||
version_counter: Arc::new(AtomicU64::new(0)),
|
||||
lifecycle_mutex: Arc::new(Mutex::new(())),
|
||||
deletion_reference_checker: std::sync::RwLock::new(None),
|
||||
audit_sink: std::sync::RwLock::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Install the sink that receives an audit record for every KMS
|
||||
/// management operation. Takes effect for services built by the next
|
||||
/// start or reconfigure.
|
||||
///
|
||||
/// Without a sink no records are built, so KMS operations are unaffected
|
||||
/// when auditing is not configured.
|
||||
pub fn set_audit_sink(&self, sink: Arc<dyn KmsAuditSink>) {
|
||||
if let Ok(mut slot) = self.audit_sink.write() {
|
||||
*slot = Some(sink);
|
||||
}
|
||||
}
|
||||
|
||||
fn audit_sink(&self) -> Option<Arc<dyn KmsAuditSink>> {
|
||||
self.audit_sink.read().ok().and_then(|slot| slot.clone())
|
||||
}
|
||||
|
||||
/// Install the reference checker consulted before the deletion worker
|
||||
/// removes an expired key. Takes effect for workers spawned by the next
|
||||
/// start or reconfigure.
|
||||
@@ -410,7 +379,7 @@ impl KmsServiceManager {
|
||||
// Note: Existing Arc references will keep the service alive until operations complete
|
||||
let state = self.state.load_full();
|
||||
if let Some(current) = state.current_service.as_ref() {
|
||||
current.shutdown_background_workers();
|
||||
current.shutdown_deletion_worker();
|
||||
}
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: state.config.clone(),
|
||||
@@ -545,19 +514,6 @@ impl KmsServiceManager {
|
||||
self.state.load().current_service.as_ref().map(|sv| sv.version)
|
||||
}
|
||||
|
||||
/// Latest synthetic probe snapshot of the running service version.
|
||||
///
|
||||
/// `None` when KMS is not running, when the backend cannot round-trip a
|
||||
/// data key, or when the probe is disabled. The read is lock-free and
|
||||
/// performs no backend I/O, so a health endpoint can consult it without
|
||||
/// turning an external KMS hiccup into probe pressure of its own; the
|
||||
/// staleness and consecutive-failure thresholds belong to the caller.
|
||||
pub fn probe_status(&self) -> Option<Arc<ProbeStatus>> {
|
||||
let state = self.state.load();
|
||||
let service_version = state.current_service.as_ref()?;
|
||||
Some(service_version.probe_worker.as_ref()?.status())
|
||||
}
|
||||
|
||||
/// Health check for the KMS service
|
||||
pub async fn health_check(&self) -> Result<bool> {
|
||||
let checked_state = self.state.load_full();
|
||||
@@ -626,19 +582,10 @@ impl KmsServiceManager {
|
||||
let backend = crate::backends::static_kms::StaticKmsBackend::new(config.clone()).await?;
|
||||
Arc::new(backend) as Arc<dyn KmsBackend>
|
||||
}
|
||||
BackendConfig::Aws(_) => {
|
||||
info!("Creating AWS KMS backend for version {}", version);
|
||||
let backend = crate::backends::aws::AwsKmsBackend::new(config.clone()).await?;
|
||||
Arc::new(backend) as Arc<dyn KmsBackend>
|
||||
}
|
||||
};
|
||||
|
||||
// Create KMS manager
|
||||
let mut kms_manager = KmsManager::new(backend, config.clone());
|
||||
if let Some(sink) = self.audit_sink() {
|
||||
kms_manager = kms_manager.with_audit_sink(sink);
|
||||
}
|
||||
let kms_manager = Arc::new(kms_manager);
|
||||
let kms_manager = Arc::new(KmsManager::new(backend, config.clone()));
|
||||
|
||||
// Create encryption service
|
||||
let encryption_service = Arc::new(ObjectEncryptionService::new((*kms_manager).clone()));
|
||||
@@ -649,7 +596,6 @@ impl KmsServiceManager {
|
||||
manager: kms_manager,
|
||||
credential_task,
|
||||
deletion_worker: None,
|
||||
probe_worker: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -663,13 +609,9 @@ impl KmsServiceManager {
|
||||
|
||||
fn publish_running(&self, config: KmsConfig, mut service_version: ServiceVersion) {
|
||||
if let Some(previous) = self.state.load().current_service.as_ref() {
|
||||
previous.shutdown_background_workers();
|
||||
previous.shutdown_deletion_worker();
|
||||
}
|
||||
service_version.deletion_worker = self.spawn_deletion_worker(&config, &service_version);
|
||||
// Started at publish time for the same reason as the deletion worker:
|
||||
// a start or reconfigure candidate that never becomes current must not
|
||||
// leak a running task or publish probe results nobody asked for.
|
||||
service_version.probe_worker = spawn_probe_worker(service_version.manager.backend()).map(Arc::new);
|
||||
self.state.store(Arc::new(RuntimeState {
|
||||
config: Some(config),
|
||||
status: KmsServiceStatus::Running,
|
||||
@@ -687,13 +629,7 @@ impl KmsServiceManager {
|
||||
return None;
|
||||
}
|
||||
let cancel = CancellationToken::new();
|
||||
let worker = DeletionWorker::new(
|
||||
backend,
|
||||
config.default_key_id.clone(),
|
||||
self.deletion_reference_checker(),
|
||||
config.backend.as_str(),
|
||||
)
|
||||
.with_audit_sink(self.audit_sink());
|
||||
let worker = DeletionWorker::new(backend, config.default_key_id.clone(), self.deletion_reference_checker());
|
||||
let task = worker.spawn(cancel.clone());
|
||||
Some(Arc::new(DeletionWorkerHandle {
|
||||
cancel,
|
||||
@@ -1044,49 +980,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn probe_worker_follows_the_service_lifecycle() {
|
||||
use tempfile::TempDir;
|
||||
|
||||
let key_dir = TempDir::new().expect("create local KMS directory");
|
||||
let mut config = KmsConfig::local(key_dir.path().to_path_buf());
|
||||
config.allow_insecure_dev_defaults = true;
|
||||
let manager = KmsServiceManager::new();
|
||||
manager.configure(config).await.expect("configure local KMS");
|
||||
manager.start().await.expect("start local KMS");
|
||||
|
||||
let first_probe = manager
|
||||
.state
|
||||
.load()
|
||||
.current_service
|
||||
.as_ref()
|
||||
.expect("running service")
|
||||
.probe_worker
|
||||
.clone()
|
||||
.expect("a backend with a data key round trip must run a probe");
|
||||
assert!(!first_probe.is_cancelled());
|
||||
assert!(manager.probe_status().is_some(), "a running service must publish probe status");
|
||||
|
||||
// Replacing the service version replaces (and cancels) its probe.
|
||||
manager.restart().await.expect("restart");
|
||||
assert!(first_probe.is_cancelled(), "replaced version's probe must be cancelled");
|
||||
let second_probe = manager
|
||||
.state
|
||||
.load()
|
||||
.current_service
|
||||
.as_ref()
|
||||
.expect("running service")
|
||||
.probe_worker
|
||||
.clone()
|
||||
.expect("restarted service must run a fresh probe");
|
||||
assert!(!second_probe.is_cancelled());
|
||||
|
||||
// Stopping the service stops its probe and withdraws the snapshot.
|
||||
manager.stop().await.expect("stop");
|
||||
assert!(second_probe.is_cancelled(), "stop must cancel the probe worker");
|
||||
assert!(manager.probe_status().is_none(), "a stopped service must publish no status");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconfigure_allows_safe_local_runtime_settings_only() {
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -484,20 +484,6 @@ impl OperationContext {
|
||||
}
|
||||
}
|
||||
|
||||
/// Principal recorded for work the server starts on its own behalf.
|
||||
///
|
||||
/// Namespaced so it can never collide with an access key or an IAM ARN,
|
||||
/// which keeps "no human did this" distinguishable from "we lost track of
|
||||
/// who did this" in the audit trail.
|
||||
pub const INTERNAL_PRINCIPAL: &'static str = "rustfs:internal";
|
||||
|
||||
/// Context for an operation with no authenticated caller, such as
|
||||
/// background maintenance or a call made while serving a data-plane
|
||||
/// request that carries its own audit entry.
|
||||
pub fn internal() -> Self {
|
||||
Self::new(Self::INTERNAL_PRINCIPAL.to_string())
|
||||
}
|
||||
|
||||
/// Add additional context
|
||||
///
|
||||
/// # Arguments
|
||||
|
||||
@@ -22,8 +22,6 @@ mod pattern_rules_test;
|
||||
#[cfg(test)]
|
||||
mod pattern_test;
|
||||
mod rules_map;
|
||||
#[cfg(test)]
|
||||
mod rules_map_test;
|
||||
mod subscriber_index;
|
||||
mod subscriber_snapshot;
|
||||
mod target_id_set;
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regressions for rule matching against the KMS event namespace.
|
||||
//!
|
||||
//! KMS events (backlog#1583) are appended to `EventName` for the audit sink,
|
||||
//! but they must stay invisible to bucket notification rules: a subscriber
|
||||
//! asking for `s3:ObjectCreated:*` — or for everything — must never receive
|
||||
//! key management activity.
|
||||
|
||||
use super::RulesMap;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
|
||||
const KMS_EVENTS: &[EventName] = &[
|
||||
EventName::KmsKeyCreated,
|
||||
EventName::KmsKeyRotated,
|
||||
EventName::KmsKeyEnabled,
|
||||
EventName::KmsKeyDisabled,
|
||||
EventName::KmsKeyDeletionScheduled,
|
||||
EventName::KmsKeyDeletionCancelled,
|
||||
EventName::KmsKeyDeleted,
|
||||
EventName::KmsKeyAccessed,
|
||||
];
|
||||
|
||||
fn test_target() -> TargetID {
|
||||
TargetID::new("primary".to_string(), "webhook".to_string())
|
||||
}
|
||||
|
||||
fn rules_for(events: &[EventName]) -> RulesMap {
|
||||
let mut rules = RulesMap::new();
|
||||
rules.add_rule_config(events, String::new(), test_target());
|
||||
rules
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s3_wildcard_subscriptions_do_not_match_kms_events() {
|
||||
let rules = rules_for(&[
|
||||
EventName::ObjectCreatedAll,
|
||||
EventName::ObjectAccessedAll,
|
||||
EventName::ObjectRemovedAll,
|
||||
EventName::ObjectTaggingAll,
|
||||
EventName::ObjectReplicationAll,
|
||||
EventName::ObjectRestoreAll,
|
||||
EventName::ObjectTransitionAll,
|
||||
EventName::LifecycleExpirationAll,
|
||||
EventName::ObjectScannerAll,
|
||||
]);
|
||||
|
||||
for event in KMS_EVENTS {
|
||||
assert!(!rules.has_subscriber(event), "{event} must not have an S3 wildcard subscriber");
|
||||
assert!(
|
||||
rules.match_rules(*event, "any/object").is_empty(),
|
||||
"{event} must not match any S3 wildcard rule"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn everything_subscription_does_not_match_kms_events() {
|
||||
let rules = rules_for(&[EventName::Everything]);
|
||||
|
||||
// Sanity: the catch-all really is subscribed to the S3 surface.
|
||||
assert!(rules.has_subscriber(&EventName::ObjectCreatedPut));
|
||||
|
||||
for event in KMS_EVENTS {
|
||||
assert!(!rules.has_subscriber(event), "{event} must stay outside the s3 catch-all");
|
||||
assert!(
|
||||
rules.match_rules(*event, "any/object").is_empty(),
|
||||
"{event} must not match the s3 catch-all rule"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kms_subscription_does_not_leak_into_s3_matching() {
|
||||
// The inverse direction: even an explicit KMS subscription must not widen
|
||||
// the mask so that S3 events start matching a KMS-only rule.
|
||||
let rules = rules_for(KMS_EVENTS);
|
||||
|
||||
for event in [
|
||||
EventName::ObjectCreatedPut,
|
||||
EventName::ObjectRemovedDelete,
|
||||
EventName::ObjectAccessedGet,
|
||||
EventName::BucketCreated,
|
||||
] {
|
||||
assert!(!rules.has_subscriber(&event), "{event} must not match a KMS-only rule");
|
||||
assert!(
|
||||
rules.match_rules(event, "any/object").is_empty(),
|
||||
"{event} must not match a KMS-only rule"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -146,7 +146,6 @@ tracing-opentelemetry = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "tracing-log", "local-time", "json"] }
|
||||
tokio = { workspace = true, features = ["sync", "rt-multi-thread", "time", "macros"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
url.workspace = true
|
||||
dial9-tokio-telemetry = { workspace = true, optional = true }
|
||||
thiserror = { workspace = true }
|
||||
zstd = { workspace = true, features = ["zstdmt"] }
|
||||
@@ -163,4 +162,3 @@ libc = { workspace = true }
|
||||
[dev-dependencies]
|
||||
tempfile = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
log = "0.4"
|
||||
|
||||
@@ -16,25 +16,22 @@
|
||||
|
||||
use crate::metrics::report::PrometheusMetric;
|
||||
use crate::metrics::schema::bucket_replication::{
|
||||
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD,
|
||||
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD, BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD,
|
||||
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD, BUCKET_REPL_LAST_HR_FAILED_COUNT_MD,
|
||||
BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD, BUCKET_REPL_LATENCY_MS_MD,
|
||||
BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD, BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_FAILED_TOTAL_MD, BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD,
|
||||
BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD, BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L,
|
||||
TARGET_ARN_L,
|
||||
BUCKET_L, BUCKET_REPL_BANDWIDTH_CURRENT_MD, BUCKET_REPL_BANDWIDTH_LIMIT_MD, BUCKET_REPL_LAST_HR_FAILED_BYTES_MD,
|
||||
BUCKET_REPL_LAST_HR_FAILED_COUNT_MD, BUCKET_REPL_LAST_MIN_FAILED_BYTES_MD, BUCKET_REPL_LAST_MIN_FAILED_COUNT_MD,
|
||||
BUCKET_REPL_LATENCY_MS_MD, BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_GET_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_GET_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_HEAD_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_HEAD_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_REQUESTS_TOTAL_MD, BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_FAILURES_MD,
|
||||
BUCKET_REPL_PROXIED_PUT_TAGGING_REQUESTS_TOTAL_MD, BUCKET_REPL_RESYNC_CANCELED_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_COMPLETED_TOTAL_MD, BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, BUCKET_REPL_RESYNC_FAILED_TOTAL_MD,
|
||||
BUCKET_REPL_RESYNC_STARTED_TOTAL_MD, BUCKET_REPL_SENT_BYTES_MD, BUCKET_REPL_SENT_COUNT_MD, BUCKET_REPL_TOTAL_FAILED_BYTES_MD,
|
||||
BUCKET_REPL_TOTAL_FAILED_COUNT_MD, OPERATION_L, RANGE_L, TARGET_ARN_L,
|
||||
};
|
||||
use std::borrow::Cow;
|
||||
|
||||
const BASE_BUCKET_REPLICATION_METRICS_PER_BUCKET: usize = 25;
|
||||
const BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET: usize = 5;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct BucketReplicationTargetStats {
|
||||
@@ -83,16 +80,6 @@ pub struct BucketReplicationStats {
|
||||
pub targets: Vec<BucketReplicationTargetStats>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub(crate) struct BucketReplicationBacklogStats {
|
||||
pub(crate) bucket: String,
|
||||
pub(crate) current_backlog_count: u64,
|
||||
pub(crate) current_backlog_bytes: u64,
|
||||
pub(crate) durable_mrf_available: bool,
|
||||
pub(crate) durable_mrf_backlog_count: u64,
|
||||
pub(crate) durable_mrf_backlog_bytes: u64,
|
||||
}
|
||||
|
||||
pub fn collect_bucket_replication_bandwidth_metrics(stats: &[BucketReplicationBandwidthStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -262,6 +249,7 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_RESYNC_DURATION_MS_TOTAL_MD, stat.resync_duration_ms as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
|
||||
for target in &stat.targets {
|
||||
let target_label: Cow<'static, str> = Cow::Owned(target.target_arn.clone());
|
||||
metrics.push(
|
||||
@@ -277,43 +265,6 @@ pub fn collect_bucket_replication_metrics(stats: &[BucketReplicationStats]) -> V
|
||||
metrics
|
||||
}
|
||||
|
||||
pub(crate) fn collect_bucket_replication_backlog_metrics(stats: &[BucketReplicationBacklogStats]) -> Vec<PrometheusMetric> {
|
||||
if stats.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut metrics = Vec::with_capacity(stats.len() * BASE_BUCKET_REPLICATION_BACKLOG_METRICS_PER_BUCKET);
|
||||
for stat in stats {
|
||||
let bucket_label: Cow<'static, str> = Cow::Owned(stat.bucket.clone());
|
||||
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD, stat.current_backlog_count as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD, stat.current_backlog_bytes as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(
|
||||
&BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD,
|
||||
if stat.durable_mrf_available { 1.0 } else { 0.0 },
|
||||
)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD, stat.durable_mrf_backlog_count as f64)
|
||||
.with_label(BUCKET_L, bucket_label.clone()),
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD, stat.durable_mrf_backlog_bytes as f64)
|
||||
.with_label(BUCKET_L, bucket_label),
|
||||
);
|
||||
}
|
||||
|
||||
metrics
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -418,56 +369,6 @@ mod tests {
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_bucket_replication_backlog_metrics() {
|
||||
let stats = vec![BucketReplicationBacklogStats {
|
||||
bucket: "b1".to_string(),
|
||||
current_backlog_count: 3,
|
||||
current_backlog_bytes: 4096,
|
||||
durable_mrf_available: true,
|
||||
durable_mrf_backlog_count: 2,
|
||||
durable_mrf_backlog_bytes: 2048,
|
||||
}];
|
||||
|
||||
let metrics = collect_bucket_replication_backlog_metrics(&stats);
|
||||
assert_eq!(metrics.len(), 5);
|
||||
|
||||
let backlog_count_name = BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == backlog_count_name
|
||||
&& metric.value == 3.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
|
||||
let backlog_bytes_name = BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == backlog_bytes_name
|
||||
&& metric.value == 4096.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
|
||||
let durable_available_name = BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == durable_available_name
|
||||
&& metric.value == 1.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
|
||||
let durable_count_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == durable_count_name
|
||||
&& metric.value == 2.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
|
||||
let durable_bytes_name = BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name();
|
||||
assert!(metrics.iter().any(|metric| {
|
||||
metric.name == durable_bytes_name
|
||||
&& metric.value == 2048.0
|
||||
&& metric.labels.iter().any(|(key, value)| *key == BUCKET_L && value == "b1")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_bucket_replication_metrics_empty() {
|
||||
let stats: Vec<BucketReplicationStats> = Vec::new();
|
||||
@@ -475,41 +376,6 @@ mod tests {
|
||||
assert!(metrics.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backlog_metrics_are_bucket_scoped_without_target_labels() {
|
||||
let stats = vec![BucketReplicationBacklogStats {
|
||||
bucket: "scope-bucket".to_string(),
|
||||
current_backlog_count: 5,
|
||||
current_backlog_bytes: 8192,
|
||||
durable_mrf_available: true,
|
||||
durable_mrf_backlog_count: 2,
|
||||
durable_mrf_backlog_bytes: 4096,
|
||||
}];
|
||||
|
||||
let metrics = collect_bucket_replication_backlog_metrics(&stats);
|
||||
let backlog_names = [
|
||||
BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD.get_full_metric_name(),
|
||||
BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD.get_full_metric_name(),
|
||||
BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD.get_full_metric_name(),
|
||||
BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD.get_full_metric_name(),
|
||||
BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD.get_full_metric_name(),
|
||||
];
|
||||
|
||||
for name in backlog_names {
|
||||
let metric = metrics
|
||||
.iter()
|
||||
.find(|metric| metric.name == name)
|
||||
.expect("backlog metric should be emitted");
|
||||
assert!(
|
||||
metric
|
||||
.labels
|
||||
.iter()
|
||||
.any(|(key, value)| *key == BUCKET_L && value == "scope-bucket")
|
||||
);
|
||||
assert!(!metric.labels.iter().any(|(key, _)| *key == TARGET_ARN_L));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_bucket_replication_bandwidth_metrics() {
|
||||
let stats = vec![BucketReplicationBandwidthStats {
|
||||
|
||||
@@ -42,7 +42,6 @@ pub mod system_process;
|
||||
|
||||
pub use audit::{AuditTargetStats, collect_audit_metrics};
|
||||
pub use bucket::{BucketStats, collect_bucket_metrics};
|
||||
pub(crate) use bucket_replication::{BucketReplicationBacklogStats, collect_bucket_replication_backlog_metrics};
|
||||
pub use bucket_replication::{
|
||||
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
|
||||
collect_bucket_replication_bandwidth_metrics, collect_bucket_replication_metrics,
|
||||
|
||||
@@ -31,9 +31,9 @@ pub use scheduler::{
|
||||
metrics_runtime_controller_snapshot, metrics_runtime_status_snapshot,
|
||||
};
|
||||
pub(crate) use storage_api::metrics::{
|
||||
BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
|
||||
StorageAdminApi, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor,
|
||||
obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
|
||||
BucketOperations, BucketOptions, ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, StorageAdminApi,
|
||||
obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
|
||||
obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
|
||||
obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
|
||||
obs_resolve_object_store_handle, obs_transition_state_handle,
|
||||
};
|
||||
|
||||
@@ -35,7 +35,6 @@ use crate::metrics::collectors::{
|
||||
ProcessMemoryStats,
|
||||
collect_audit_metrics,
|
||||
collect_bucket_metrics,
|
||||
collect_bucket_replication_backlog_metrics,
|
||||
collect_bucket_replication_bandwidth_metrics,
|
||||
collect_bucket_replication_metrics,
|
||||
collect_bucket_usage_metrics,
|
||||
@@ -95,7 +94,7 @@ use crate::metrics::schema::notification_target::{
|
||||
};
|
||||
use crate::metrics::schema::system_process::{PROCESS_EXECUTABLE_NAME_LABEL, PROCESS_PID_LABEL};
|
||||
use crate::metrics::stats_collector::{
|
||||
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_stats_bundle,
|
||||
ProcessMetricBundle, collect_bucket_replication_bandwidth_stats, collect_bucket_replication_detail_stats,
|
||||
collect_bucket_stats, collect_cluster_and_health_stats, collect_cluster_config_stats, collect_cluster_usage_metric_stats,
|
||||
collect_compression_cluster_stats, collect_disk_and_system_drive_stats, collect_erasure_set_stats,
|
||||
collect_host_network_stats, collect_iam_stats, collect_ilm_metric_stats, collect_internode_network_stats,
|
||||
@@ -1349,9 +1348,8 @@ pub fn init_metrics_runtime(token: CancellationToken) {
|
||||
// Phase-1 action: force zero for removed keys during tombstone cycles.
|
||||
metrics.extend(collect_repl_bw_zero_tombstone_metrics(&zero_tombstones));
|
||||
|
||||
let (bucket_replication, bucket_replication_backlog) = collect_bucket_replication_stats_bundle().await;
|
||||
let bucket_replication = collect_bucket_replication_detail_stats().await;
|
||||
metrics.extend(collect_bucket_replication_metrics(&bucket_replication));
|
||||
metrics.extend(collect_bucket_replication_backlog_metrics(&bucket_replication_backlog));
|
||||
let replication = collect_replication_stats().await;
|
||||
metrics.extend(collect_replication_metrics(&replication));
|
||||
report_metrics(&metrics);
|
||||
|
||||
@@ -33,11 +33,6 @@ const RESYNC_COMPLETED_TOTAL: &str = "resync_completed_total";
|
||||
const RESYNC_FAILED_TOTAL: &str = "resync_failed_total";
|
||||
const RESYNC_CANCELED_TOTAL: &str = "resync_canceled_total";
|
||||
const RESYNC_DURATION_MS_TOTAL: &str = "resync_duration_ms_total";
|
||||
const CURRENT_BACKLOG_COUNT: &str = "current_backlog_count";
|
||||
const CURRENT_BACKLOG_BYTES: &str = "current_backlog_bytes";
|
||||
const DURABLE_MRF_AVAILABLE: &str = "durable_mrf_available";
|
||||
const DURABLE_MRF_BACKLOG_COUNT: &str = "durable_mrf_backlog_count";
|
||||
const DURABLE_MRF_BACKLOG_BYTES: &str = "durable_mrf_backlog_bytes";
|
||||
|
||||
pub static BUCKET_REPL_LAST_HR_FAILED_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
@@ -84,51 +79,6 @@ pub static BUCKET_REPL_LATENCY_MS_MD: LazyLock<MetricDescriptor> = LazyLock::new
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_CURRENT_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(CURRENT_BACKLOG_BYTES),
|
||||
"Current number of bytes admitted to the in-memory replication worker queues for a bucket",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_CURRENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(CURRENT_BACKLOG_COUNT),
|
||||
"Current number of objects admitted to the in-memory replication worker queues for a bucket",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_DURABLE_MRF_AVAILABLE_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(DURABLE_MRF_AVAILABLE),
|
||||
"Whether the durable MRF backlog snapshot is available for this bucket on this node",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(DURABLE_MRF_BACKLOG_COUNT),
|
||||
"Current number of objects in the durable MRF backlog file for a bucket on this node",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_DURABLE_MRF_BACKLOG_BYTES_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::from(DURABLE_MRF_BACKLOG_BYTES),
|
||||
"Current bytes in the durable MRF backlog file for a bucket on this node",
|
||||
&[BUCKET_L],
|
||||
subsystems::BUCKET_REPLICATION,
|
||||
)
|
||||
});
|
||||
|
||||
pub static BUCKET_REPL_PROXIED_DELETE_TAGGING_REQUESTS_TOTAL_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_counter_md(
|
||||
MetricName::ProxiedDeleteTaggingRequestsTotal,
|
||||
|
||||
@@ -128,7 +128,7 @@ pub static REPLICATION_MAX_DATA_TRANSFER_RATE_MD: LazyLock<MetricDescriptor> = L
|
||||
pub static REPLICATION_RECENT_BACKLOG_COUNT_MD: LazyLock<MetricDescriptor> = LazyLock::new(|| {
|
||||
new_gauge_md(
|
||||
MetricName::ReplicationRecentBacklogCount,
|
||||
"Legacy replication backlog indicator: failed target objects plus objects currently queued on this node",
|
||||
"Total number of objects currently in replication backlog (failed plus queued)",
|
||||
&[],
|
||||
subsystems::REPLICATION,
|
||||
)
|
||||
|
||||
@@ -21,18 +21,17 @@
|
||||
//! and convert them to the Stats structs used by collectors.
|
||||
|
||||
use crate::metrics::collectors::{
|
||||
BucketReplicationBacklogStats, BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats,
|
||||
BucketStats, BucketUsageStats, ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats,
|
||||
CompressionClusterStats, CpuStats, DiskStats, DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats,
|
||||
IamStats, IlmStats, MemoryStats, NetworkStats, ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats,
|
||||
ScannerStats,
|
||||
BucketReplicationBandwidthStats, BucketReplicationStats, BucketReplicationTargetStats, BucketStats, BucketUsageStats,
|
||||
ClusterConfigStats, ClusterHealthStats, ClusterStats, ClusterUsageStats, CompressionClusterStats, CpuStats, DiskStats,
|
||||
DriveCountStats, DriveDetailedStats, ErasureSetStats, HostNetworkStats, IamStats, IlmStats, MemoryStats, NetworkStats,
|
||||
ProcessStats, ProcessStatusType, ReplicationStats, ResourceStats, ScannerStats,
|
||||
};
|
||||
use crate::metrics::runtime_sources::{ObsIlmRuntimeSnapshot, bucket_monitor_handle, iam_metrics_snapshot, ilm_runtime_snapshot};
|
||||
use crate::metrics::{
|
||||
BucketOperations, BucketOptions, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore, StorageAdminApi,
|
||||
obs_bucket_replication_stats_snapshot, obs_get_quota_config, obs_get_total_usable_capacity,
|
||||
obs_get_total_usable_capacity_free, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
|
||||
obs_replication_site_stats_snapshot, obs_resolve_object_store_handle,
|
||||
BucketOperations, BucketOptions, ObsEcstoreResult, ObsStore, StorageAdminApi, obs_bucket_replication_stats_snapshot,
|
||||
obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
|
||||
obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
|
||||
obs_resolve_object_store_handle,
|
||||
};
|
||||
use crate::node_identity::current_local_node_identity;
|
||||
use chrono::Utc;
|
||||
@@ -193,65 +192,49 @@ async fn obs_ilm_runtime_snapshot() -> ObsIlmRuntimeSnapshot {
|
||||
ilm_runtime_snapshot().await
|
||||
}
|
||||
|
||||
async fn obs_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>) {
|
||||
let snapshots = obs_bucket_replication_stats_snapshot().await;
|
||||
let mut detail_stats = Vec::with_capacity(snapshots.len());
|
||||
let mut backlog_stats = Vec::with_capacity(snapshots.len());
|
||||
|
||||
for stats in snapshots {
|
||||
backlog_stats.push(BucketReplicationBacklogStats {
|
||||
bucket: stats.bucket.clone(),
|
||||
current_backlog_count: stats.current_backlog_count,
|
||||
current_backlog_bytes: stats.current_backlog_bytes,
|
||||
durable_mrf_available: stats.durable_mrf_available,
|
||||
durable_mrf_backlog_count: stats.durable_mrf_backlog_count,
|
||||
durable_mrf_backlog_bytes: stats.durable_mrf_backlog_bytes,
|
||||
});
|
||||
detail_stats.push(bucket_replication_detail_from_snapshot(stats));
|
||||
}
|
||||
|
||||
(detail_stats, backlog_stats)
|
||||
}
|
||||
|
||||
fn bucket_replication_detail_from_snapshot(stats: ObsBucketReplicationStatsSnapshot) -> BucketReplicationStats {
|
||||
BucketReplicationStats {
|
||||
bucket: stats.bucket,
|
||||
total_failed_bytes: stats.total_failed_bytes,
|
||||
total_failed_count: stats.total_failed_count,
|
||||
last_min_failed_bytes: stats.last_min_failed_bytes,
|
||||
last_min_failed_count: stats.last_min_failed_count,
|
||||
last_hour_failed_bytes: stats.last_hour_failed_bytes,
|
||||
last_hour_failed_count: stats.last_hour_failed_count,
|
||||
sent_bytes: stats.sent_bytes,
|
||||
sent_count: stats.sent_count,
|
||||
proxied_get_requests_total: stats.proxied_get_requests_total,
|
||||
proxied_get_requests_failures: stats.proxied_get_requests_failures,
|
||||
proxied_head_requests_total: stats.proxied_head_requests_total,
|
||||
proxied_head_requests_failures: stats.proxied_head_requests_failures,
|
||||
proxied_put_requests_total: stats.proxied_put_requests_total,
|
||||
proxied_put_requests_failures: stats.proxied_put_requests_failures,
|
||||
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
|
||||
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
|
||||
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
|
||||
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
|
||||
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
|
||||
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
|
||||
resync_started_count: stats.resync_started_count,
|
||||
resync_completed_count: stats.resync_completed_count,
|
||||
resync_failed_count: stats.resync_failed_count,
|
||||
resync_canceled_count: stats.resync_canceled_count,
|
||||
resync_duration_ms: stats.resync_duration_ms,
|
||||
targets: stats
|
||||
.targets
|
||||
.into_iter()
|
||||
.map(|target| BucketReplicationTargetStats {
|
||||
target_arn: target.target_arn,
|
||||
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
|
||||
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
|
||||
latency_ms: target.latency_ms,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
async fn obs_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
|
||||
obs_bucket_replication_stats_snapshot()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(|stats| BucketReplicationStats {
|
||||
bucket: stats.bucket,
|
||||
total_failed_bytes: stats.total_failed_bytes,
|
||||
total_failed_count: stats.total_failed_count,
|
||||
last_min_failed_bytes: stats.last_min_failed_bytes,
|
||||
last_min_failed_count: stats.last_min_failed_count,
|
||||
last_hour_failed_bytes: stats.last_hour_failed_bytes,
|
||||
last_hour_failed_count: stats.last_hour_failed_count,
|
||||
sent_bytes: stats.sent_bytes,
|
||||
sent_count: stats.sent_count,
|
||||
proxied_get_requests_total: stats.proxied_get_requests_total,
|
||||
proxied_get_requests_failures: stats.proxied_get_requests_failures,
|
||||
proxied_head_requests_total: stats.proxied_head_requests_total,
|
||||
proxied_head_requests_failures: stats.proxied_head_requests_failures,
|
||||
proxied_put_requests_total: stats.proxied_put_requests_total,
|
||||
proxied_put_requests_failures: stats.proxied_put_requests_failures,
|
||||
proxied_put_tagging_requests_total: stats.proxied_put_tagging_requests_total,
|
||||
proxied_put_tagging_requests_failures: stats.proxied_put_tagging_requests_failures,
|
||||
proxied_get_tagging_requests_total: stats.proxied_get_tagging_requests_total,
|
||||
proxied_get_tagging_requests_failures: stats.proxied_get_tagging_requests_failures,
|
||||
proxied_delete_tagging_requests_total: stats.proxied_delete_tagging_requests_total,
|
||||
proxied_delete_tagging_requests_failures: stats.proxied_delete_tagging_requests_failures,
|
||||
resync_started_count: stats.resync_started_count,
|
||||
resync_completed_count: stats.resync_completed_count,
|
||||
resync_failed_count: stats.resync_failed_count,
|
||||
resync_canceled_count: stats.resync_canceled_count,
|
||||
resync_duration_ms: stats.resync_duration_ms,
|
||||
targets: stats
|
||||
.targets
|
||||
.into_iter()
|
||||
.map(|target| BucketReplicationTargetStats {
|
||||
target_arn: target.target_arn,
|
||||
bandwidth_limit_bytes_per_sec: target.bandwidth_limit_bytes_per_sec,
|
||||
current_bandwidth_bytes_per_sec: target.current_bandwidth_bytes_per_sec,
|
||||
latency_ms: target.latency_ms,
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn obs_site_replication_stats() -> ReplicationStats {
|
||||
@@ -543,16 +526,7 @@ pub fn collect_bucket_replication_bandwidth_stats() -> Vec<BucketReplicationBand
|
||||
|
||||
/// Collect bucket and target level replication stats from the global replication runtime.
|
||||
pub async fn collect_bucket_replication_detail_stats() -> Vec<BucketReplicationStats> {
|
||||
obs_bucket_replication_stats_snapshot()
|
||||
.await
|
||||
.into_iter()
|
||||
.map(bucket_replication_detail_from_snapshot)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn collect_bucket_replication_stats_bundle() -> (Vec<BucketReplicationStats>, Vec<BucketReplicationBacklogStats>)
|
||||
{
|
||||
obs_bucket_replication_stats_bundle().await
|
||||
obs_bucket_replication_detail_stats().await
|
||||
}
|
||||
|
||||
/// Collect site-level replication stats from the global replication runtime.
|
||||
|
||||
@@ -12,14 +12,11 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::bucket::bandwidth::monitor::Monitor as ObsBucketBandwidthMonitor;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata_sys::get_quota_config as obs_get_quota_config;
|
||||
use rustfs_ecstore::api::bucket::replication::{
|
||||
DurableMrfBucketBacklog, durable_mrf_backlog_summary_snapshot, get_global_replication_stats,
|
||||
};
|
||||
use rustfs_ecstore::api::bucket::replication::get_global_replication_stats;
|
||||
pub(crate) use rustfs_ecstore::api::capacity::{
|
||||
get_total_usable_capacity as obs_get_total_usable_capacity,
|
||||
get_total_usable_capacity_free as obs_get_total_usable_capacity_free,
|
||||
@@ -71,50 +68,9 @@ pub(crate) struct ObsBucketReplicationStatsSnapshot {
|
||||
pub(crate) resync_failed_count: u64,
|
||||
pub(crate) resync_canceled_count: u64,
|
||||
pub(crate) resync_duration_ms: u64,
|
||||
pub(crate) current_backlog_count: u64,
|
||||
pub(crate) current_backlog_bytes: u64,
|
||||
pub(crate) durable_mrf_available: bool,
|
||||
pub(crate) durable_mrf_backlog_count: u64,
|
||||
pub(crate) durable_mrf_backlog_bytes: u64,
|
||||
pub(crate) targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
struct ObsBucketReplicationRuntimeSnapshot {
|
||||
total_failed_bytes: u64,
|
||||
total_failed_count: u64,
|
||||
last_min_failed_bytes: u64,
|
||||
last_min_failed_count: u64,
|
||||
last_hour_failed_bytes: u64,
|
||||
last_hour_failed_count: u64,
|
||||
sent_bytes: u64,
|
||||
sent_count: u64,
|
||||
resync_started_count: u64,
|
||||
resync_completed_count: u64,
|
||||
resync_failed_count: u64,
|
||||
resync_canceled_count: u64,
|
||||
resync_duration_ms: u64,
|
||||
current_backlog_count: u64,
|
||||
current_backlog_bytes: u64,
|
||||
targets: Vec<ObsBucketReplicationTargetStatsSnapshot>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
struct ObsBucketReplicationProxySnapshot {
|
||||
proxied_get_requests_total: u64,
|
||||
proxied_get_requests_failures: u64,
|
||||
proxied_head_requests_total: u64,
|
||||
proxied_head_requests_failures: u64,
|
||||
proxied_put_requests_total: u64,
|
||||
proxied_put_requests_failures: u64,
|
||||
proxied_put_tagging_requests_total: u64,
|
||||
proxied_put_tagging_requests_failures: u64,
|
||||
proxied_get_tagging_requests_total: u64,
|
||||
proxied_get_tagging_requests_failures: u64,
|
||||
proxied_delete_tagging_requests_total: u64,
|
||||
proxied_delete_tagging_requests_failures: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq)]
|
||||
pub(crate) struct ObsReplicationSiteStatsSnapshot {
|
||||
pub(crate) average_active_workers: f64,
|
||||
@@ -146,85 +102,59 @@ fn replication_backlog_count(failed_counts: impl Iterator<Item = i64>, queued_co
|
||||
failed_backlog.saturating_add(i64_to_u64_floor_zero(queued_count))
|
||||
}
|
||||
|
||||
fn bucket_replication_stats_snapshot_from_parts(
|
||||
bucket: String,
|
||||
runtime: ObsBucketReplicationRuntimeSnapshot,
|
||||
proxy: ObsBucketReplicationProxySnapshot,
|
||||
durable_mrf_available: bool,
|
||||
durable_bucket: DurableMrfBucketBacklog,
|
||||
) -> ObsBucketReplicationStatsSnapshot {
|
||||
ObsBucketReplicationStatsSnapshot {
|
||||
bucket,
|
||||
total_failed_bytes: runtime.total_failed_bytes,
|
||||
total_failed_count: runtime.total_failed_count,
|
||||
last_min_failed_bytes: runtime.last_min_failed_bytes,
|
||||
last_min_failed_count: runtime.last_min_failed_count,
|
||||
last_hour_failed_bytes: runtime.last_hour_failed_bytes,
|
||||
last_hour_failed_count: runtime.last_hour_failed_count,
|
||||
sent_bytes: runtime.sent_bytes,
|
||||
sent_count: runtime.sent_count,
|
||||
proxied_get_requests_total: proxy.proxied_get_requests_total,
|
||||
proxied_get_requests_failures: proxy.proxied_get_requests_failures,
|
||||
proxied_head_requests_total: proxy.proxied_head_requests_total,
|
||||
proxied_head_requests_failures: proxy.proxied_head_requests_failures,
|
||||
proxied_put_requests_total: proxy.proxied_put_requests_total,
|
||||
proxied_put_requests_failures: proxy.proxied_put_requests_failures,
|
||||
proxied_put_tagging_requests_total: proxy.proxied_put_tagging_requests_total,
|
||||
proxied_put_tagging_requests_failures: proxy.proxied_put_tagging_requests_failures,
|
||||
proxied_get_tagging_requests_total: proxy.proxied_get_tagging_requests_total,
|
||||
proxied_get_tagging_requests_failures: proxy.proxied_get_tagging_requests_failures,
|
||||
proxied_delete_tagging_requests_total: proxy.proxied_delete_tagging_requests_total,
|
||||
proxied_delete_tagging_requests_failures: proxy.proxied_delete_tagging_requests_failures,
|
||||
resync_started_count: runtime.resync_started_count,
|
||||
resync_completed_count: runtime.resync_completed_count,
|
||||
resync_failed_count: runtime.resync_failed_count,
|
||||
resync_canceled_count: runtime.resync_canceled_count,
|
||||
resync_duration_ms: runtime.resync_duration_ms,
|
||||
current_backlog_count: runtime.current_backlog_count,
|
||||
current_backlog_bytes: runtime.current_backlog_bytes,
|
||||
durable_mrf_available,
|
||||
durable_mrf_backlog_count: durable_bucket.count,
|
||||
durable_mrf_backlog_bytes: durable_bucket.bytes,
|
||||
targets: runtime.targets,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketReplicationStatsSnapshot> {
|
||||
let stats = get_global_replication_stats();
|
||||
let all_bucket_stats = if let Some(stats) = &stats {
|
||||
stats.get_all().await
|
||||
} else {
|
||||
HashMap::new()
|
||||
let Some(stats) = get_global_replication_stats() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let durable_mrf_summary = if obs_resolve_object_store_handle().is_some() {
|
||||
durable_mrf_backlog_summary_snapshot()
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
let durable_mrf_available = durable_mrf_summary.available;
|
||||
let durable_buckets = durable_mrf_summary
|
||||
.buckets
|
||||
.into_iter()
|
||||
.map(|bucket| (bucket.bucket.clone(), bucket))
|
||||
.collect::<HashMap<String, DurableMrfBucketBacklog>>();
|
||||
let mut bucket_names = Vec::with_capacity(all_bucket_stats.len().saturating_add(durable_buckets.len()));
|
||||
bucket_names.extend(all_bucket_stats.keys().cloned());
|
||||
bucket_names.extend(
|
||||
durable_buckets
|
||||
.keys()
|
||||
.filter(|bucket| !all_bucket_stats.contains_key(*bucket))
|
||||
.cloned(),
|
||||
);
|
||||
let mut buckets = Vec::with_capacity(bucket_names.len());
|
||||
|
||||
for bucket in bucket_names {
|
||||
let bucket_stats = all_bucket_stats.get(&bucket);
|
||||
let proxy = if let Some(stats) = &stats {
|
||||
stats.get_proxy_stats(&bucket).await
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
let proxy = ObsBucketReplicationProxySnapshot {
|
||||
let all_bucket_stats = stats.get_all().await;
|
||||
let mut buckets = Vec::with_capacity(all_bucket_stats.len());
|
||||
|
||||
for (bucket, bucket_stats) in all_bucket_stats {
|
||||
let proxy = stats.get_proxy_stats(&bucket).await;
|
||||
let mut total_failed_bytes = 0u64;
|
||||
let mut total_failed_count = 0u64;
|
||||
let mut last_min_failed_bytes = 0u64;
|
||||
let mut last_min_failed_count = 0u64;
|
||||
let mut last_hour_failed_bytes = 0u64;
|
||||
let mut last_hour_failed_count = 0u64;
|
||||
let mut sent_bytes = 0u64;
|
||||
let mut sent_count = 0u64;
|
||||
let mut targets = Vec::with_capacity(bucket_stats.stats.len());
|
||||
|
||||
for (target_arn, target_stats) in bucket_stats.stats {
|
||||
total_failed_bytes = total_failed_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
|
||||
total_failed_count = total_failed_count.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
|
||||
|
||||
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
|
||||
last_min_failed_bytes = last_min_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_min.size));
|
||||
last_min_failed_count = last_min_failed_count.saturating_add(i64_to_u64_floor_zero(last_min.count));
|
||||
|
||||
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
|
||||
last_hour_failed_bytes = last_hour_failed_bytes.saturating_add(i64_to_u64_floor_zero(last_hour.size));
|
||||
last_hour_failed_count = last_hour_failed_count.saturating_add(i64_to_u64_floor_zero(last_hour.count));
|
||||
|
||||
sent_bytes = sent_bytes.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
|
||||
sent_count = sent_count.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
|
||||
|
||||
targets.push(ObsBucketReplicationTargetStatsSnapshot {
|
||||
target_arn,
|
||||
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
|
||||
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
|
||||
latency_ms: target_stats.latency.curr,
|
||||
});
|
||||
}
|
||||
|
||||
buckets.push(ObsBucketReplicationStatsSnapshot {
|
||||
bucket,
|
||||
total_failed_bytes,
|
||||
total_failed_count,
|
||||
last_min_failed_bytes,
|
||||
last_min_failed_count,
|
||||
last_hour_failed_bytes,
|
||||
last_hour_failed_count,
|
||||
sent_bytes,
|
||||
sent_count,
|
||||
proxied_get_requests_total: i64_to_u64_floor_zero(proxy.get_total),
|
||||
proxied_get_requests_failures: i64_to_u64_floor_zero(proxy.get_failed),
|
||||
proxied_head_requests_total: i64_to_u64_floor_zero(proxy.head_total),
|
||||
@@ -237,67 +167,13 @@ pub(crate) async fn obs_bucket_replication_stats_snapshot() -> Vec<ObsBucketRepl
|
||||
proxied_get_tagging_requests_failures: i64_to_u64_floor_zero(proxy.get_tag_failed),
|
||||
proxied_delete_tagging_requests_total: i64_to_u64_floor_zero(proxy.delete_tag_total),
|
||||
proxied_delete_tagging_requests_failures: i64_to_u64_floor_zero(proxy.delete_tag_failed),
|
||||
};
|
||||
let mut runtime = ObsBucketReplicationRuntimeSnapshot {
|
||||
targets: Vec::with_capacity(bucket_stats.map(|stats| stats.stats.len()).unwrap_or(0)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Some(bucket_stats) = bucket_stats {
|
||||
for (target_arn, target_stats) in &bucket_stats.stats {
|
||||
runtime.total_failed_bytes = runtime
|
||||
.total_failed_bytes
|
||||
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.size));
|
||||
runtime.total_failed_count = runtime
|
||||
.total_failed_count
|
||||
.saturating_add(i64_to_u64_floor_zero(target_stats.fail_stats.count));
|
||||
|
||||
let last_min = target_stats.fail_stats.recent_since(Duration::from_secs(60));
|
||||
runtime.last_min_failed_bytes = runtime
|
||||
.last_min_failed_bytes
|
||||
.saturating_add(i64_to_u64_floor_zero(last_min.size));
|
||||
runtime.last_min_failed_count = runtime
|
||||
.last_min_failed_count
|
||||
.saturating_add(i64_to_u64_floor_zero(last_min.count));
|
||||
|
||||
let last_hour = target_stats.fail_stats.recent_since(Duration::from_secs(60 * 60));
|
||||
runtime.last_hour_failed_bytes = runtime
|
||||
.last_hour_failed_bytes
|
||||
.saturating_add(i64_to_u64_floor_zero(last_hour.size));
|
||||
runtime.last_hour_failed_count = runtime
|
||||
.last_hour_failed_count
|
||||
.saturating_add(i64_to_u64_floor_zero(last_hour.count));
|
||||
|
||||
runtime.sent_bytes = runtime
|
||||
.sent_bytes
|
||||
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_size));
|
||||
runtime.sent_count = runtime
|
||||
.sent_count
|
||||
.saturating_add(i64_to_u64_floor_zero(target_stats.replicated_count));
|
||||
|
||||
runtime.targets.push(ObsBucketReplicationTargetStatsSnapshot {
|
||||
target_arn: target_arn.clone(),
|
||||
bandwidth_limit_bytes_per_sec: i64_to_u64_floor_zero(target_stats.bandwidth_limit_bytes_per_sec),
|
||||
current_bandwidth_bytes_per_sec: target_stats.current_bandwidth_bytes_per_sec,
|
||||
latency_ms: target_stats.latency.curr,
|
||||
});
|
||||
}
|
||||
runtime.resync_started_count = i64_to_u64_floor_zero(bucket_stats.resync_started_count);
|
||||
runtime.resync_completed_count = i64_to_u64_floor_zero(bucket_stats.resync_completed_count);
|
||||
runtime.resync_failed_count = i64_to_u64_floor_zero(bucket_stats.resync_failed_count);
|
||||
runtime.resync_canceled_count = i64_to_u64_floor_zero(bucket_stats.resync_canceled_count);
|
||||
runtime.resync_duration_ms = i64_to_u64_floor_zero(bucket_stats.resync_duration_ms);
|
||||
runtime.current_backlog_count = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.count);
|
||||
runtime.current_backlog_bytes = i64_to_u64_floor_zero(bucket_stats.q_stat.curr.bytes);
|
||||
}
|
||||
let durable_bucket = durable_buckets.get(&bucket).cloned().unwrap_or_default();
|
||||
buckets.push(bucket_replication_stats_snapshot_from_parts(
|
||||
bucket,
|
||||
runtime,
|
||||
proxy,
|
||||
durable_mrf_available,
|
||||
durable_bucket,
|
||||
));
|
||||
resync_started_count: i64_to_u64_floor_zero(bucket_stats.resync_started_count),
|
||||
resync_completed_count: i64_to_u64_floor_zero(bucket_stats.resync_completed_count),
|
||||
resync_failed_count: i64_to_u64_floor_zero(bucket_stats.resync_failed_count),
|
||||
resync_canceled_count: i64_to_u64_floor_zero(bucket_stats.resync_canceled_count),
|
||||
resync_duration_ms: i64_to_u64_floor_zero(bucket_stats.resync_duration_ms),
|
||||
targets,
|
||||
});
|
||||
}
|
||||
|
||||
buckets
|
||||
@@ -367,97 +243,15 @@ mod tests {
|
||||
fn replication_backlog_count_floors_negative_failed_and_queue_values() {
|
||||
assert_eq!(replication_backlog_count([-3, 4].into_iter(), -2), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_backlog_count_keeps_legacy_failed_backlog_semantics() {
|
||||
assert_eq!(replication_backlog_count([9].into_iter(), 0), 9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_replication_snapshot_maps_runtime_and_durable_backlog() {
|
||||
let snapshot = bucket_replication_stats_snapshot_from_parts(
|
||||
"runtime-bucket".to_string(),
|
||||
ObsBucketReplicationRuntimeSnapshot {
|
||||
current_backlog_count: 3,
|
||||
current_backlog_bytes: 4096,
|
||||
resync_failed_count: 2,
|
||||
..Default::default()
|
||||
},
|
||||
ObsBucketReplicationProxySnapshot {
|
||||
proxied_get_requests_total: 7,
|
||||
proxied_get_requests_failures: 1,
|
||||
..Default::default()
|
||||
},
|
||||
true,
|
||||
DurableMrfBucketBacklog {
|
||||
bucket: "runtime-bucket".to_string(),
|
||||
count: 5,
|
||||
bytes: 8192,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(snapshot.bucket, "runtime-bucket");
|
||||
assert_eq!(snapshot.current_backlog_count, 3);
|
||||
assert_eq!(snapshot.current_backlog_bytes, 4096);
|
||||
assert!(snapshot.durable_mrf_available);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_count, 5);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_bytes, 8192);
|
||||
assert_eq!(snapshot.resync_failed_count, 2);
|
||||
assert_eq!(snapshot.proxied_get_requests_total, 7);
|
||||
assert_eq!(snapshot.proxied_get_requests_failures, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_replication_snapshot_reports_durable_only_bucket() {
|
||||
let snapshot = bucket_replication_stats_snapshot_from_parts(
|
||||
"durable-only".to_string(),
|
||||
ObsBucketReplicationRuntimeSnapshot::default(),
|
||||
ObsBucketReplicationProxySnapshot::default(),
|
||||
true,
|
||||
DurableMrfBucketBacklog {
|
||||
bucket: "durable-only".to_string(),
|
||||
count: 11,
|
||||
bytes: 2048,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(snapshot.bucket, "durable-only");
|
||||
assert_eq!(snapshot.current_backlog_count, 0);
|
||||
assert!(snapshot.durable_mrf_available);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_count, 11);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_bytes, 2048);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_replication_snapshot_preserves_durable_mrf_unavailable_state() {
|
||||
let snapshot = bucket_replication_stats_snapshot_from_parts(
|
||||
"runtime-only".to_string(),
|
||||
ObsBucketReplicationRuntimeSnapshot {
|
||||
current_backlog_count: 1,
|
||||
current_backlog_bytes: 512,
|
||||
..Default::default()
|
||||
},
|
||||
ObsBucketReplicationProxySnapshot::default(),
|
||||
false,
|
||||
DurableMrfBucketBacklog::default(),
|
||||
);
|
||||
|
||||
assert_eq!(snapshot.current_backlog_count, 1);
|
||||
assert_eq!(snapshot.current_backlog_bytes, 512);
|
||||
assert!(!snapshot.durable_mrf_available);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_count, 0);
|
||||
assert_eq!(snapshot.durable_mrf_backlog_bytes, 0);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod metrics {
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, BucketOptions, StorageAdminApi};
|
||||
|
||||
pub(crate) use super::{
|
||||
ObsBucketBandwidthMonitor, ObsBucketReplicationStatsSnapshot, ObsEcstoreResult, ObsStore,
|
||||
obs_bucket_replication_stats_snapshot, obs_expiry_state_handle, obs_get_global_bucket_monitor, obs_get_quota_config,
|
||||
obs_get_total_usable_capacity, obs_get_total_usable_capacity_free, obs_is_disk_compression_enabled,
|
||||
obs_load_compression_total_from_memory, obs_load_data_usage_from_backend, obs_replication_site_stats_snapshot,
|
||||
obs_resolve_object_store_handle, obs_transition_state_handle,
|
||||
ObsBucketBandwidthMonitor, ObsEcstoreResult, ObsStore, obs_bucket_replication_stats_snapshot, obs_expiry_state_handle,
|
||||
obs_get_global_bucket_monitor, obs_get_quota_config, obs_get_total_usable_capacity, obs_get_total_usable_capacity_free,
|
||||
obs_is_disk_compression_enabled, obs_load_compression_total_from_memory, obs_load_data_usage_from_backend,
|
||||
obs_replication_site_stats_snapshot, obs_resolve_object_store_handle, obs_transition_state_handle,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -18,26 +18,7 @@
|
||||
//! used across different logging backends (stdout, file, OpenTelemetry).
|
||||
|
||||
use std::env;
|
||||
use tracing::Metadata;
|
||||
use tracing_subscriber::{
|
||||
EnvFilter,
|
||||
filter::{FilterFn, LevelFilter, filter_fn},
|
||||
};
|
||||
|
||||
/// Pyroscope emits raw reqwest errors from its background session manager.
|
||||
/// Those errors can include the configured endpoint, so they must never reach
|
||||
/// a RustFS logging sink even when an operator enables verbose dependency logs.
|
||||
pub(super) fn is_pyroscope_dependency_target(target: &str) -> bool {
|
||||
target == "pyroscope" || target.starts_with("pyroscope::") || target.starts_with("Pyroscope::")
|
||||
}
|
||||
|
||||
fn allows_non_pyroscope_dependency_event(metadata: &Metadata<'_>) -> bool {
|
||||
!is_pyroscope_dependency_target(metadata.target())
|
||||
}
|
||||
|
||||
pub(super) fn pyroscope_log_filter() -> FilterFn {
|
||||
filter_fn(allows_non_pyroscope_dependency_event)
|
||||
}
|
||||
use tracing_subscriber::{EnvFilter, filter::LevelFilter};
|
||||
|
||||
/// Build an `EnvFilter` from the given log level string.
|
||||
///
|
||||
|
||||
@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
|
||||
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
|
||||
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
|
||||
use crate::telemetry::filter::build_env_filter;
|
||||
use crate::telemetry::rolling::{RollingAppender, Rotation};
|
||||
use metrics::counter;
|
||||
use rustfs_config::observability::{
|
||||
@@ -319,7 +319,6 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(fmt_layer)
|
||||
.try_init()
|
||||
@@ -432,7 +431,6 @@ fn init_file_logging_internal(
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(file_layer)
|
||||
.with(stdout_layer)
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
use crate::cleaner::types::FileMatchMode;
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::set_observability_metric_enabled;
|
||||
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter};
|
||||
use crate::telemetry::filter::build_env_filter;
|
||||
use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
|
||||
use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
|
||||
use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
|
||||
@@ -68,7 +68,7 @@ use std::{fs, io::IsTerminal, time::Duration};
|
||||
use tracing::{info, warn};
|
||||
use tracing_error::ErrorLayer;
|
||||
use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
|
||||
use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
use tracing_subscriber::{Layer, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
|
||||
const GET_OBJECT_DURATION_HISTOGRAM_METRICS: &[&str] = &[
|
||||
"rustfs_io_get_object_request_duration_seconds",
|
||||
@@ -83,26 +83,6 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
|
||||
];
|
||||
|
||||
#[cfg(all(
|
||||
feature = "pyroscope",
|
||||
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
|
||||
))]
|
||||
const REDACTED_PROFILING_ENDPOINT: &str = "[redacted]";
|
||||
|
||||
#[cfg(all(
|
||||
feature = "pyroscope",
|
||||
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
|
||||
))]
|
||||
fn log_profiler_failure(result: &'static str, error_kind: &'static str) {
|
||||
warn!(
|
||||
backend = "pyroscope",
|
||||
endpoint = REDACTED_PROFILING_ENDPOINT,
|
||||
result,
|
||||
error_kind,
|
||||
"Profiling export agent initialization failed"
|
||||
);
|
||||
}
|
||||
|
||||
/// Initialize the full OpenTelemetry HTTP pipeline (traces + metrics + logs).
|
||||
///
|
||||
/// This function is invoked when at least one OTLP endpoint has been
|
||||
@@ -178,7 +158,9 @@ pub(super) fn init_observability_http(
|
||||
logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?;
|
||||
|
||||
// Build bridge to capture `tracing` events.
|
||||
otel_bridge = logger_provider.as_ref().map(OpenTelemetryTracingBridge::new);
|
||||
otel_bridge = logger_provider
|
||||
.as_ref()
|
||||
.map(|p| OpenTelemetryTracingBridge::new(p).with_filter(build_env_filter(logger_level, None)));
|
||||
|
||||
// No separate formatting layer is added here; when OTLP logging is
|
||||
// active, the OpenTelemetry bridge is the authoritative sink for
|
||||
@@ -274,7 +256,6 @@ pub(super) fn init_observability_http(
|
||||
let filter = build_env_filter(logger_level, None);
|
||||
tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(file_layer_opt)
|
||||
.with(stdout_layer_opt)
|
||||
@@ -550,11 +531,6 @@ pub(super) fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
|
||||
return None;
|
||||
};
|
||||
|
||||
if url::Url::parse(endpoint).is_err() {
|
||||
log_profiler_failure("profiling_endpoint_invalid", "invalid_endpoint");
|
||||
return None;
|
||||
}
|
||||
|
||||
// Configure Pyroscope Agent
|
||||
let backend = pprof_backend(PprofConfig::default(), BackendConfig::default());
|
||||
let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
|
||||
@@ -566,16 +542,28 @@ pub(super) fn init_profiler(config: &OtelConfig) -> Option<ProfilingAgent> {
|
||||
.build()
|
||||
{
|
||||
Ok(agent) => agent,
|
||||
Err(_) => {
|
||||
log_profiler_failure("profiling_agent_build_failed", "build");
|
||||
Err(err) => {
|
||||
warn!(
|
||||
backend = "pyroscope",
|
||||
endpoint,
|
||||
result = "profiling_agent_build_failed",
|
||||
error = %err,
|
||||
"Profiling export agent initialization failed"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
match agent.start() {
|
||||
Ok(agent) => Some(agent),
|
||||
Err(_) => {
|
||||
log_profiler_failure("profiling_agent_start_failed", "start");
|
||||
Err(err) => {
|
||||
warn!(
|
||||
backend = "pyroscope",
|
||||
endpoint,
|
||||
result = "profiling_agent_start_failed",
|
||||
error = ?err,
|
||||
"Profiling export agent failed to start"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -662,34 +650,6 @@ fn resolve_signal_timeout(common_timeout_millis: Option<u64>, signal_timeout_mil
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::env;
|
||||
use std::io::{self, Write};
|
||||
use std::process::Command;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
const LOG_TRACER_CHILD_ENV: &str = "RUSTFS_OBS_LOG_TRACER_CHILD";
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestWriter(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl Write for TestWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.0.lock().expect("lock test log buffer").extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for TestWriter {
|
||||
type Writer = Self;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Valid ratios should produce trace-id-ratio sampling.
|
||||
@@ -806,107 +766,4 @@ mod tests {
|
||||
assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.first(), Some(&0.0001));
|
||||
assert_eq!(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS.last(), Some(&10.0));
|
||||
}
|
||||
|
||||
#[cfg(all(
|
||||
feature = "pyroscope",
|
||||
any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
|
||||
))]
|
||||
#[test]
|
||||
fn test_init_profiler_invalid_endpoint_redacts_sensitive_components() {
|
||||
let endpoint = "https://profile-user:profile-token@10.24.0.5:invalid/private/profiles?access_token=query-secret";
|
||||
let buffer = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = TestWriter(Arc::clone(&buffer));
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_writer(writer)
|
||||
.finish();
|
||||
|
||||
let config = OtelConfig {
|
||||
profiling_export_enabled: Some(true),
|
||||
profiling_endpoint: Some(endpoint.to_string()),
|
||||
..OtelConfig::default()
|
||||
};
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || assert!(init_profiler(&config).is_none()));
|
||||
|
||||
let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
|
||||
assert!(rendered.contains(REDACTED_PROFILING_ENDPOINT));
|
||||
assert!(rendered.contains("error_kind=\"invalid_endpoint\""));
|
||||
for leaked in [
|
||||
"profile-user",
|
||||
"profile-token",
|
||||
"10.24.0.5",
|
||||
"private/profiles",
|
||||
"query-secret",
|
||||
] {
|
||||
assert!(!rendered.contains(leaked), "profiling log leaked {leaked}: {rendered}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pyroscope_upload_failure_targets_stay_filtered_when_env_filter_enables_them() {
|
||||
let endpoint = "https://profile-user:profile-token@10.24.0.5/private/profiles?access_token=query-secret";
|
||||
let buffer = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = TestWriter(Arc::clone(&buffer));
|
||||
let env_filter = tracing_subscriber::EnvFilter::new("pyroscope=trace")
|
||||
.add_directive("trace".parse().expect("parse trace filter directive"))
|
||||
.add_directive("Pyroscope::Session=trace".parse().expect("parse Pyroscope filter directive"));
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_writer(writer),
|
||||
);
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
tracing::error!(target: "pyroscope::session", "SessionManager - Failed to send session: {endpoint}");
|
||||
tracing::error!(target: "Pyroscope::Session", "SessionManager - Failed to send session: {endpoint}");
|
||||
});
|
||||
|
||||
let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
|
||||
assert!(rendered.is_empty(), "filtered Pyroscope upload failure reached a log sink: {rendered}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pyroscope_log_facade_upload_failure_is_filtered() {
|
||||
let endpoint = "https://profile-user:profile-token@10.24.0.5/private/profiles?access_token=query-secret";
|
||||
if env::var_os(LOG_TRACER_CHILD_ENV).is_some() {
|
||||
tracing_subscriber::registry()
|
||||
.with(build_env_filter("info", None))
|
||||
.with(pyroscope_log_filter())
|
||||
.with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
|
||||
.try_init()
|
||||
.expect("install isolated LogTracer subscriber");
|
||||
log::error!(target: "pyroscope::session", "SessionManager - Failed to send session: {endpoint}");
|
||||
log::error!(target: "Pyroscope::Session", "SessionManager - Failed to send session: {endpoint}");
|
||||
return;
|
||||
}
|
||||
|
||||
let output = Command::new(env::current_exe().expect("resolve test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"telemetry::otel::tests::test_pyroscope_log_facade_upload_failure_is_filtered",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(LOG_TRACER_CHILD_ENV, "1")
|
||||
.env("RUST_LOG", "trace,pyroscope=trace,Pyroscope::Session=trace")
|
||||
.output()
|
||||
.expect("run isolated LogTracer test process");
|
||||
assert!(output.status.success(), "isolated LogTracer test failed: {output:?}");
|
||||
|
||||
let rendered = format!("{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr));
|
||||
for leaked in [
|
||||
"profile-user",
|
||||
"profile-token",
|
||||
"10.24.0.5",
|
||||
"private/profiles",
|
||||
"query-secret",
|
||||
] {
|
||||
assert!(!rendered.contains(leaked), "LogTracer path leaked {leaked}: {rendered}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ workspace = true
|
||||
default = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/reqwest-0-13",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-credentials/hotpath",
|
||||
"rustfs-crypto/hotpath",
|
||||
@@ -79,7 +80,6 @@ pollster.workspace = true
|
||||
proptest = "1"
|
||||
test-case.workspace = true
|
||||
temp-env = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["fmt"] }
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
+15
-225
@@ -30,16 +30,17 @@ impl Args {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_configured() -> bool {
|
||||
env::var_os(ENV_POLICY_PLUGIN_OPA_URL).is_some_and(|url| !url.is_empty())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AuthZPlugin {
|
||||
client: reqwest::Client,
|
||||
client: OpaHttpClient,
|
||||
args: Args,
|
||||
}
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
type OpaHttpClient = hotpath::wrap::reqwest::Client;
|
||||
#[cfg(not(feature = "hotpath"))]
|
||||
type OpaHttpClient = reqwest::Client;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum OpaConfigError {
|
||||
#[error("Missing required env var: {0}")]
|
||||
@@ -58,44 +59,6 @@ pub enum OpaConfigError {
|
||||
Connection(reqwest::Error),
|
||||
}
|
||||
|
||||
impl OpaConfigError {
|
||||
pub fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Self::MissingRequiredEnv(_) => "missing_required_env",
|
||||
Self::InvalidEnvVars(_) => "invalid_env_vars",
|
||||
Self::EnvRead { .. } => "env_read_failed",
|
||||
Self::InvalidStatus(_) => "invalid_status",
|
||||
Self::Connection(_) => "connection_failed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn redact_opa_connection_error(error: reqwest::Error) -> OpaConfigError {
|
||||
OpaConfigError::Connection(error.without_url())
|
||||
}
|
||||
|
||||
fn opa_request_error_kind(error: &reqwest::Error) -> &'static str {
|
||||
if error.is_timeout() {
|
||||
"timeout"
|
||||
} else if error.is_connect() {
|
||||
"connect"
|
||||
} else if error.is_request() {
|
||||
"request"
|
||||
} else if error.is_body() {
|
||||
"body"
|
||||
} else if error.is_decode() {
|
||||
"decode"
|
||||
} else if error.is_status() {
|
||||
"status"
|
||||
} else if error.is_builder() {
|
||||
"builder"
|
||||
} else if error.is_redirect() {
|
||||
"redirect"
|
||||
} else {
|
||||
"other"
|
||||
}
|
||||
}
|
||||
|
||||
fn check() -> Result<(), OpaConfigError> {
|
||||
let env_list = env::vars();
|
||||
let mut candidate = HashMap::new();
|
||||
@@ -123,14 +86,9 @@ async fn validate(config: &Args) -> Result<(), OpaConfigError> {
|
||||
.timeout(Duration::from_secs(5))
|
||||
.connect_timeout(Duration::from_secs(1))
|
||||
.build()
|
||||
.map_err(redact_opa_connection_error)?;
|
||||
.map_err(OpaConfigError::Connection)?;
|
||||
|
||||
let mut request = client.post(&config.url);
|
||||
if !config.auth_token.is_empty() {
|
||||
request = request.header("Authorization", format!("Bearer {}", config.auth_token));
|
||||
}
|
||||
|
||||
match request.send().await {
|
||||
match client.post(&config.url).send().await {
|
||||
Ok(resp) => {
|
||||
match resp.status() {
|
||||
reqwest::StatusCode::OK => {
|
||||
@@ -142,7 +100,7 @@ async fn validate(config: &Args) -> Result<(), OpaConfigError> {
|
||||
};
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(redact_opa_connection_error(err));
|
||||
return Err(OpaConfigError::Connection(err));
|
||||
}
|
||||
};
|
||||
Ok(())
|
||||
@@ -188,6 +146,9 @@ impl AuthZPlugin {
|
||||
reqwest::Client::new()
|
||||
});
|
||||
|
||||
#[cfg(feature = "hotpath")]
|
||||
let client = hotpath::http!(client, label = "Policy::OPA");
|
||||
|
||||
Self { client, args: config }
|
||||
}
|
||||
|
||||
@@ -203,13 +164,7 @@ impl AuthZPlugin {
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
error!(
|
||||
component = "policy",
|
||||
subsystem = "opa",
|
||||
result = "request_rejected",
|
||||
status = %status,
|
||||
"OPA request rejected"
|
||||
);
|
||||
error!("OPA returned non-success status: {}", status);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -219,25 +174,13 @@ impl AuthZPlugin {
|
||||
OpaResponseEnum::AllowResult(response) => response.result.allow,
|
||||
},
|
||||
Err(err) => {
|
||||
error!(
|
||||
component = "policy",
|
||||
subsystem = "opa",
|
||||
result = "response_decode_failed",
|
||||
error_kind = opa_request_error_kind(&err),
|
||||
"OPA response decoding failed"
|
||||
);
|
||||
error!("Error parsing OPA response: {:?}", err);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
component = "policy",
|
||||
subsystem = "opa",
|
||||
result = "request_failed",
|
||||
error_kind = opa_request_error_kind(&err),
|
||||
"OPA request failed"
|
||||
);
|
||||
error!("Error sending request to OPA: {:?}", err);
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -306,35 +249,8 @@ enum OpaResponseEnum {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use temp_env;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestWriter(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl Write for TestWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.0.lock().expect("lock test log buffer").extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for TestWriter {
|
||||
type Writer = Self;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_reqwest_client(_: &reqwest::Client) {}
|
||||
|
||||
#[test]
|
||||
fn test_check_valid_config() {
|
||||
// Use temp_env to temporarily set environment variables
|
||||
@@ -420,130 +336,4 @@ mod tests {
|
||||
};
|
||||
assert!(!args_disabled.enable());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sensitive_opa_endpoints_never_use_hotpath_http_wrapper() {
|
||||
let endpoints = [
|
||||
"https://profile-user:profile-token@10.24.0.5/private/opa?access_token=query-secret",
|
||||
"https://service.internal.example/private/path?token=another-secret",
|
||||
];
|
||||
|
||||
for endpoint in endpoints {
|
||||
let plugin = AuthZPlugin::new(Args {
|
||||
url: endpoint.to_string(),
|
||||
auth_token: "opa-auth-token".to_string(),
|
||||
});
|
||||
|
||||
assert_reqwest_client(&plugin.client);
|
||||
assert_eq!(plugin.args.url, endpoint);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_opa_connection_error_removes_sensitive_endpoint() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test listener");
|
||||
let port = listener.local_addr().expect("read local test listener address").port();
|
||||
let server = std::thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("accept local test request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let read = stream.read(&mut request).expect("read local test request");
|
||||
assert!(read > 0, "local test client must send an HTTP request");
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("write local test response");
|
||||
});
|
||||
let endpoint = format!("http://profile-user:profile-token@127.0.0.1:{port}/private/opa?access_token=query-secret");
|
||||
let runtime = tokio::runtime::Runtime::new().expect("build tokio test runtime");
|
||||
let error = runtime.block_on(async {
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.build()
|
||||
.expect("build local test client")
|
||||
.get(&endpoint)
|
||||
.send()
|
||||
.await
|
||||
.expect("send local test request")
|
||||
.error_for_status()
|
||||
.expect_err("test response should be an HTTP error")
|
||||
});
|
||||
server.join().expect("join local test server");
|
||||
assert!(error.url().is_some(), "fixture must carry the endpoint before redaction");
|
||||
|
||||
let OpaConfigError::Connection(error) = redact_opa_connection_error(error) else {
|
||||
panic!("expected a redacted OPA connection error");
|
||||
};
|
||||
let rendered = error.to_string();
|
||||
assert!(error.url().is_none());
|
||||
for leaked in ["profile-user", "profile-token", "127.0.0.1", "private/opa", "query-secret"] {
|
||||
assert!(!rendered.contains(leaked), "redacted error leaked {leaked}: {rendered}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_allowed_rejection_log_redacts_sensitive_endpoint() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind local test listener");
|
||||
let port = listener.local_addr().expect("read local test listener address").port();
|
||||
let server = std::thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().expect("accept local test request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let read = stream.read(&mut request).expect("read local test request");
|
||||
assert!(read > 0, "local test client must send an HTTP request");
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.expect("write local test response");
|
||||
});
|
||||
let endpoint = format!("http://opa-user:opa-token@127.0.0.1:{port}/private/allow?access_token=query-secret");
|
||||
let plugin = AuthZPlugin::new(Args {
|
||||
url: endpoint,
|
||||
auth_token: "authorization-secret".to_string(),
|
||||
});
|
||||
let groups = None;
|
||||
let conditions = HashMap::new();
|
||||
let claims = HashMap::new();
|
||||
let args = PArgs {
|
||||
account: "account",
|
||||
groups: &groups,
|
||||
action: crate::policy::action::Action::None,
|
||||
bucket: "bucket",
|
||||
conditions: &conditions,
|
||||
is_owner: false,
|
||||
object: "object",
|
||||
claims: &claims,
|
||||
deny_only: false,
|
||||
};
|
||||
let buffer = Arc::new(Mutex::new(Vec::new()));
|
||||
let writer = TestWriter(Arc::clone(&buffer));
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_writer(writer)
|
||||
.finish();
|
||||
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build tokio test runtime");
|
||||
assert!(!runtime.block_on(plugin.is_allowed(&args)));
|
||||
});
|
||||
server.join().expect("join local test server");
|
||||
|
||||
let rendered = String::from_utf8(buffer.lock().expect("lock test log buffer").clone()).expect("decode test log");
|
||||
assert!(!rendered.is_empty(), "OPA request failure did not reach the test log sink");
|
||||
assert!(
|
||||
rendered.contains("request_rejected"),
|
||||
"OPA request failure did not retain its stable result category"
|
||||
);
|
||||
assert!(rendered.contains("500 Internal Server Error"));
|
||||
for leaked in [
|
||||
"opa-user",
|
||||
"opa-token",
|
||||
"127.0.0.1",
|
||||
"private/allow",
|
||||
"query-secret",
|
||||
"authorization-secret",
|
||||
] {
|
||||
assert!(!rendered.contains(leaked), "OPA request log leaked {leaked}: {rendered}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1171,12 +1171,6 @@ pub struct SignalServiceResponse {
|
||||
pub error_info: ::core::option::Option<::prost::alloc::string::String>,
|
||||
#[prost(uint32, tag = "3")]
|
||||
pub protocol_version: u32,
|
||||
/// Redacted fingerprint of the KMS configuration this node is running.
|
||||
/// Only set for the "kms" dynamic config subsystem, and left unset when the
|
||||
/// responder has no KMS configuration. Peers built before that subsystem
|
||||
/// reject the signal outright rather than answering without this field.
|
||||
#[prost(string, optional, tag = "4")]
|
||||
pub config_fingerprint: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ScannerActivityRequest {
|
||||
|
||||
@@ -817,11 +817,6 @@ message SignalServiceResponse {
|
||||
bool success = 1;
|
||||
optional string error_info = 2;
|
||||
uint32 protocol_version = 3;
|
||||
// Redacted fingerprint of the KMS configuration this node is running.
|
||||
// Only set for the "kms" dynamic config subsystem, and left unset when the
|
||||
// responder has no KMS configuration. Peers built before that subsystem
|
||||
// reject the signal outright rather than answering without this field.
|
||||
optional string config_fingerprint = 4;
|
||||
}
|
||||
|
||||
message ScannerActivityRequest {
|
||||
|
||||
@@ -241,16 +241,6 @@ impl InQueueStats {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn add_current(&self, bytes: i64, count: i64) {
|
||||
self.now_bytes.fetch_add(bytes.max(0), Ordering::Relaxed);
|
||||
self.now_count.fetch_add(count.max(0), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn subtract_current(&self, bytes: i64, count: i64) {
|
||||
saturating_atomic_sub(&self.now_bytes, bytes.max(0));
|
||||
saturating_atomic_sub(&self.now_count, count.max(0));
|
||||
}
|
||||
|
||||
pub fn get_current_bytes(&self) -> i64 {
|
||||
self.now_bytes.load(Ordering::Relaxed)
|
||||
}
|
||||
@@ -260,14 +250,6 @@ impl InQueueStats {
|
||||
}
|
||||
}
|
||||
|
||||
fn saturating_atomic_sub(value: &AtomicI64, delta: i64) {
|
||||
if delta == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let _ = value.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_sub(delta).max(0)));
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct InQueueMetric {
|
||||
pub curr: InQueueStats,
|
||||
@@ -377,18 +359,6 @@ impl QueueCache {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn inc(&mut self, bucket: &str, size: i64) {
|
||||
let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
|
||||
stats.curr.add_current(size, 1);
|
||||
self.sr_queue_stats.curr.add_current(size, 1);
|
||||
}
|
||||
|
||||
pub fn dec(&mut self, bucket: &str, size: i64) {
|
||||
let stats = self.bucket_stats.entry(bucket.to_string()).or_default();
|
||||
stats.curr.subtract_current(size, 1);
|
||||
self.sr_queue_stats.curr.subtract_current(size, 1);
|
||||
}
|
||||
|
||||
pub fn update(&mut self) {
|
||||
let observed_at = Instant::now();
|
||||
self.sr_queue_stats.observe(observed_at);
|
||||
@@ -838,10 +808,12 @@ mod tests {
|
||||
#[test]
|
||||
fn in_queue_metric_observe_updates_rolling_stats() {
|
||||
let mut metric = InQueueMetric::default();
|
||||
metric.curr.add_current(128, 4);
|
||||
metric.curr.now_bytes.store(128, Ordering::Relaxed);
|
||||
metric.curr.now_count.store(4, Ordering::Relaxed);
|
||||
metric.observe(Instant::now());
|
||||
|
||||
metric.curr.add_current(128, 2);
|
||||
metric.curr.now_bytes.store(256, Ordering::Relaxed);
|
||||
metric.curr.now_count.store(6, Ordering::Relaxed);
|
||||
metric.observe(Instant::now());
|
||||
|
||||
assert_eq!(metric.curr.bytes, 256);
|
||||
@@ -852,21 +824,6 @@ mod tests {
|
||||
assert_eq!(metric.last_minute.count, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queue_cache_decrement_saturates_at_zero() {
|
||||
let mut cache = QueueCache::new();
|
||||
cache.inc("bucket", 128);
|
||||
cache.dec("bucket", 512);
|
||||
cache.dec("bucket", 1);
|
||||
|
||||
let bucket = cache.get_bucket_stats("bucket");
|
||||
let site = cache.get_site_stats();
|
||||
assert_eq!(bucket.curr.count, 0);
|
||||
assert_eq!(bucket.curr.bytes, 0);
|
||||
assert_eq!(site.curr.count, 0);
|
||||
assert_eq!(site.curr.bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_stats_recent_since_tracks_windows() {
|
||||
let mut stats = FailStats::default();
|
||||
|
||||
@@ -86,20 +86,6 @@ pub enum EventName {
|
||||
ObjectRemovedAbortMultipartUpload,
|
||||
ObjectCreatedCreateMultipartUpload,
|
||||
ObjectRemovedDeleteObjects,
|
||||
|
||||
// KMS management-plane events. They travel to the audit sink only and are
|
||||
// never produced by the bucket notification path, so no compound `s3:`
|
||||
// event expands to them. New variants must keep being appended here: the
|
||||
// discriminant of every preceding variant is a `mask()` bit position, and
|
||||
// inserting in the middle would silently renumber existing bits.
|
||||
KmsKeyCreated,
|
||||
KmsKeyRotated,
|
||||
KmsKeyEnabled,
|
||||
KmsKeyDisabled,
|
||||
KmsKeyDeletionScheduled,
|
||||
KmsKeyDeletionCancelled,
|
||||
KmsKeyDeleted,
|
||||
KmsKeyAccessed,
|
||||
}
|
||||
|
||||
// Single event type sequential array for Everything.expand()
|
||||
@@ -200,17 +186,6 @@ impl EventName {
|
||||
"s3:Scanner:LargeVersions" => Ok(EventName::ScannerLargeVersions),
|
||||
"s3:Scanner:BigPrefix" => Ok(EventName::ScannerBigPrefix),
|
||||
"s3:Scanner:*" => Ok(EventName::ObjectScannerAll),
|
||||
// KMS events use their own namespace so a `s3:` wildcard in a bucket
|
||||
// notification config can never select them. They still round-trip
|
||||
// because audit entries are persisted and replayed by the store targets.
|
||||
"kms:Key:Created" => Ok(EventName::KmsKeyCreated),
|
||||
"kms:Key:Rotated" => Ok(EventName::KmsKeyRotated),
|
||||
"kms:Key:Enabled" => Ok(EventName::KmsKeyEnabled),
|
||||
"kms:Key:Disabled" => Ok(EventName::KmsKeyDisabled),
|
||||
"kms:Key:DeletionScheduled" => Ok(EventName::KmsKeyDeletionScheduled),
|
||||
"kms:Key:DeletionCancelled" => Ok(EventName::KmsKeyDeletionCancelled),
|
||||
"kms:Key:Deleted" => Ok(EventName::KmsKeyDeleted),
|
||||
"kms:Key:Accessed" => Ok(EventName::KmsKeyAccessed),
|
||||
// `Everything` has no string representation (`as_str` yields ""), so it
|
||||
// cannot be parsed back from a string. Every other variant round-trips.
|
||||
_ => Err(ParseEventNameError(s.to_string())),
|
||||
@@ -276,14 +251,6 @@ impl EventName {
|
||||
EventName::ObjectRemovedAbortMultipartUpload => "s3:ObjectRemoved:AbortMultipartUpload",
|
||||
EventName::ObjectCreatedCreateMultipartUpload => "s3:ObjectCreated:CreateMultipartUpload",
|
||||
EventName::ObjectRemovedDeleteObjects => "s3:ObjectRemoved:DeleteObjects",
|
||||
EventName::KmsKeyCreated => "kms:Key:Created",
|
||||
EventName::KmsKeyRotated => "kms:Key:Rotated",
|
||||
EventName::KmsKeyEnabled => "kms:Key:Enabled",
|
||||
EventName::KmsKeyDisabled => "kms:Key:Disabled",
|
||||
EventName::KmsKeyDeletionScheduled => "kms:Key:DeletionScheduled",
|
||||
EventName::KmsKeyDeletionCancelled => "kms:Key:DeletionCancelled",
|
||||
EventName::KmsKeyDeleted => "kms:Key:Deleted",
|
||||
EventName::KmsKeyAccessed => "kms:Key:Accessed",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -390,25 +357,6 @@ impl EventName {
|
||||
| EventName::ObjectRemovedDeleteObjects
|
||||
)
|
||||
}
|
||||
|
||||
/// Returns `true` for KMS management-plane events.
|
||||
///
|
||||
/// These are audit-only: they are never emitted through the bucket
|
||||
/// notification pipeline, and no `s3:` event selector expands to them.
|
||||
#[inline]
|
||||
pub fn is_kms(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
EventName::KmsKeyCreated
|
||||
| EventName::KmsKeyRotated
|
||||
| EventName::KmsKeyEnabled
|
||||
| EventName::KmsKeyDisabled
|
||||
| EventName::KmsKeyDeletionScheduled
|
||||
| EventName::KmsKeyDeletionCancelled
|
||||
| EventName::KmsKeyDeleted
|
||||
| EventName::KmsKeyAccessed
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the S3 notification event schema version for a given event.
|
||||
@@ -622,26 +570,6 @@ mod tests {
|
||||
EventName::ObjectRemovedAbortMultipartUpload,
|
||||
EventName::ObjectCreatedCreateMultipartUpload,
|
||||
EventName::ObjectRemovedDeleteObjects,
|
||||
EventName::KmsKeyCreated,
|
||||
EventName::KmsKeyRotated,
|
||||
EventName::KmsKeyEnabled,
|
||||
EventName::KmsKeyDisabled,
|
||||
EventName::KmsKeyDeletionScheduled,
|
||||
EventName::KmsKeyDeletionCancelled,
|
||||
EventName::KmsKeyDeleted,
|
||||
EventName::KmsKeyAccessed,
|
||||
];
|
||||
|
||||
/// Every KMS management-plane event.
|
||||
const KMS_EVENT_NAMES: &[EventName] = &[
|
||||
EventName::KmsKeyCreated,
|
||||
EventName::KmsKeyRotated,
|
||||
EventName::KmsKeyEnabled,
|
||||
EventName::KmsKeyDisabled,
|
||||
EventName::KmsKeyDeletionScheduled,
|
||||
EventName::KmsKeyDeletionCancelled,
|
||||
EventName::KmsKeyDeleted,
|
||||
EventName::KmsKeyAccessed,
|
||||
];
|
||||
|
||||
/// Regression for backlog#965: `mask()` used to recurse forever for the
|
||||
@@ -754,54 +682,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// KMS events are audit-plane only. No `s3:` selector — including the
|
||||
/// catch-all `Everything` and every compound "All" type — may share a bit
|
||||
/// with them, otherwise a bucket notification rule would silently start
|
||||
/// matching KMS activity.
|
||||
#[test]
|
||||
fn test_kms_event_masks_are_disjoint_from_every_s3_selector() {
|
||||
let s3_selectors: Vec<EventName> = ALL_EVENT_NAMES.iter().copied().filter(|ev| !ev.is_kms()).collect();
|
||||
|
||||
let mut seen = 0u64;
|
||||
for kms in KMS_EVENT_NAMES {
|
||||
let mask = kms.mask();
|
||||
assert_ne!(mask, 0, "KMS event {kms} must have a non-zero mask");
|
||||
assert_eq!(seen & mask, 0, "KMS event {kms} mask overlaps another KMS event");
|
||||
seen |= mask;
|
||||
|
||||
for s3 in &s3_selectors {
|
||||
assert_eq!(s3.mask() & mask, 0, "KMS event {kms} mask collides with S3 selector {s3}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// KMS event names must live in their own namespace so that neither a
|
||||
/// `s3:` prefix filter nor an `s3:...:*` wildcard can select them.
|
||||
#[test]
|
||||
fn test_kms_event_names_are_outside_the_s3_namespace() {
|
||||
for ev in KMS_EVENT_NAMES {
|
||||
assert!(ev.is_kms(), "{ev} should be classified as a KMS event");
|
||||
assert!(ev.as_str().starts_with("kms:"), "unexpected KMS event name {:?}", ev.as_str());
|
||||
assert_eq!(EventName::parse(ev.as_str()).as_ref(), Ok(ev), "KMS event {ev} must round-trip");
|
||||
assert_eq!(ev.expand(), vec![*ev], "KMS event {ev} must expand to itself only");
|
||||
}
|
||||
|
||||
for ev in ALL_EVENT_NAMES.iter().filter(|ev| !ev.is_kms()) {
|
||||
assert!(!ev.as_str().starts_with("kms:"), "{ev} must not claim the KMS namespace");
|
||||
}
|
||||
}
|
||||
|
||||
/// `mask()` shifts by `discriminant - 1`, so the enum may hold at most 64
|
||||
/// mask-bearing variants. Appending past that overflows the shift instead
|
||||
/// of failing loudly, so keep the budget check next to the variants.
|
||||
#[test]
|
||||
fn test_mask_bit_budget_is_not_exhausted() {
|
||||
for ev in ALL_EVENT_NAMES {
|
||||
let value = *ev as u32;
|
||||
assert!(value <= 64, "{ev} has discriminant {value}; mask() only has 64 bits to hand out");
|
||||
}
|
||||
}
|
||||
|
||||
/// `Everything` must cover every sequential single-type bit.
|
||||
#[test]
|
||||
fn test_everything_mask_covers_all_single_types() {
|
||||
|
||||
@@ -75,7 +75,7 @@ const DATA_SCANNER_COMPACT_LEAST_OBJECT: usize = 500;
|
||||
const DATA_SCANNER_COMPACT_AT_CHILDREN: usize = 10000;
|
||||
const DATA_SCANNER_COMPACT_AT_FOLDERS: usize = DATA_SCANNER_COMPACT_AT_CHILDREN / 4;
|
||||
const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
|
||||
const SCANNER_LIST_PATH_RAW_STALL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const SCANNER_LIST_PATH_RAW_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32;
|
||||
const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30);
|
||||
const DEFAULT_HEAL_OBJECT_SELECT_PROB: u32 = 1024;
|
||||
@@ -100,21 +100,6 @@ static SCANNER_INLINE_HEAL_WARN_ONCE: Once = Once::new();
|
||||
static SCANNER_INLINE_HEAL_METRICS_ONCE: Once = Once::new();
|
||||
static SCANNER_ALERT_METRICS_ONCE: Once = Once::new();
|
||||
|
||||
#[cfg(test)]
|
||||
type ListPathRawTimeoutSnapshot = (bool, Option<Duration>, Option<Duration>);
|
||||
|
||||
fn scanner_abandoned_child_list_options() -> ListPathRawOptions {
|
||||
// A complete heal walk scales with bucket size and may legitimately take
|
||||
// longer than a fixed wall-clock budget. Keep the total duration unbounded;
|
||||
// Retain the scanner's per-read stall budget and keep cancellation controlled
|
||||
// by the scanner cycle token.
|
||||
ListPathRawOptions {
|
||||
skip_walkdir_total_timeout: true,
|
||||
walkdir_stall_timeout: Some(SCANNER_LIST_PATH_RAW_STALL_TIMEOUT),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn data_usage_update_dir_cycles() -> u32 {
|
||||
rustfs_utils::get_env_u32(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, DATA_USAGE_UPDATE_DIR_CYCLES)
|
||||
}
|
||||
@@ -1296,8 +1281,6 @@ pub struct FolderScanner {
|
||||
skip_heal: Arc<std::sync::atomic::AtomicBool>,
|
||||
local_disk: Arc<Disk>,
|
||||
pending_heals_changed: bool,
|
||||
#[cfg(test)]
|
||||
list_path_raw_options_observer: Option<mpsc::UnboundedSender<ListPathRawTimeoutSnapshot>>,
|
||||
}
|
||||
|
||||
impl FolderScanner {
|
||||
@@ -2427,79 +2410,75 @@ impl FolderScanner {
|
||||
let bucket_clone = bucket.clone();
|
||||
let prefix_clone = prefix.clone();
|
||||
let child_ctx_clone = child_ctx.clone();
|
||||
#[cfg(test)]
|
||||
let list_path_raw_options_observer = self.list_path_raw_options_observer.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let options = ListPathRawOptions {
|
||||
disks,
|
||||
bucket: bucket_clone.clone(),
|
||||
path: prefix_clone.clone(),
|
||||
recursive: true,
|
||||
report_not_found: true,
|
||||
min_disks: disks_quorum,
|
||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
|
||||
let entry_name = entry.name.clone();
|
||||
let agreed_tx = agreed_tx.clone();
|
||||
Box::pin(async move {
|
||||
if let Err(e) = agreed_tx.send(entry_name).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
entry = %entry.name,
|
||||
state = "list_path_agreed_send_failed",
|
||||
error = %e,
|
||||
"Scanner list_path_raw agreed callback failed"
|
||||
);
|
||||
}
|
||||
})
|
||||
})),
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||
let partial_tx = partial_tx.clone();
|
||||
Box::pin(async move {
|
||||
if let Err(e) = partial_tx.send(entries).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
state = "list_path_partial_send_failed",
|
||||
error = %e,
|
||||
"Scanner list_path_raw partial callback failed"
|
||||
);
|
||||
}
|
||||
})
|
||||
})),
|
||||
finished: Some(Box::new(move |errs: &[Option<DiskError>]| {
|
||||
let finished_tx = finished_tx.clone();
|
||||
let errs_clone = errs.to_vec();
|
||||
Box::pin(async move {
|
||||
if let Err(e) = finished_tx.send(errs_clone).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
state = "list_path_finished_send_failed",
|
||||
error = %e,
|
||||
"Scanner list_path_raw finished callback failed"
|
||||
);
|
||||
}
|
||||
})
|
||||
})),
|
||||
..scanner_abandoned_child_list_options()
|
||||
};
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = list_path_raw_options_observer {
|
||||
let _ = observer.send((
|
||||
options.skip_walkdir_total_timeout,
|
||||
options.walkdir_timeout,
|
||||
options.walkdir_stall_timeout,
|
||||
));
|
||||
}
|
||||
if let Err(e) = list_path_raw(child_ctx_clone.clone(), options).await {
|
||||
if let Err(e) = list_path_raw(
|
||||
child_ctx_clone.clone(),
|
||||
ListPathRawOptions {
|
||||
disks,
|
||||
bucket: bucket_clone.clone(),
|
||||
path: prefix_clone.clone(),
|
||||
recursive: true,
|
||||
report_not_found: true,
|
||||
min_disks: disks_quorum,
|
||||
walkdir_timeout: Some(SCANNER_LIST_PATH_RAW_TIMEOUT),
|
||||
walkdir_stall_timeout: Some(SCANNER_LIST_PATH_RAW_TIMEOUT),
|
||||
agreed: Some(Box::new(move |entry: MetaCacheEntry| {
|
||||
let entry_name = entry.name.clone();
|
||||
let agreed_tx = agreed_tx.clone();
|
||||
Box::pin(async move {
|
||||
if let Err(e) = agreed_tx.send(entry_name).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
entry = %entry.name,
|
||||
state = "list_path_agreed_send_failed",
|
||||
error = %e,
|
||||
"Scanner list_path_raw agreed callback failed"
|
||||
);
|
||||
}
|
||||
})
|
||||
})),
|
||||
partial: Some(Box::new(move |entries: MetaCacheEntries, _: &[Option<DiskError>]| {
|
||||
let partial_tx = partial_tx.clone();
|
||||
Box::pin(async move {
|
||||
if let Err(e) = partial_tx.send(entries).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
state = "list_path_partial_send_failed",
|
||||
error = %e,
|
||||
"Scanner list_path_raw partial callback failed"
|
||||
);
|
||||
}
|
||||
})
|
||||
})),
|
||||
finished: Some(Box::new(move |errs: &[Option<DiskError>]| {
|
||||
let finished_tx = finished_tx.clone();
|
||||
let errs_clone = errs.to_vec();
|
||||
Box::pin(async move {
|
||||
if let Err(e) = finished_tx.send(errs_clone).await {
|
||||
error!(
|
||||
target: "rustfs::scanner::folder",
|
||||
event = EVENT_SCANNER_FOLDER_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_FOLDER,
|
||||
state = "list_path_finished_send_failed",
|
||||
error = %e,
|
||||
"Scanner list_path_raw finished callback failed"
|
||||
);
|
||||
}
|
||||
})
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
if is_missing_path_disk_error(&e) {
|
||||
debug!(
|
||||
target: "rustfs::scanner::folder",
|
||||
@@ -2841,8 +2820,6 @@ pub async fn scan_data_folder(
|
||||
skip_heal,
|
||||
local_disk,
|
||||
pending_heals_changed: false,
|
||||
#[cfg(test)]
|
||||
list_path_raw_options_observer: None,
|
||||
};
|
||||
|
||||
// Check if context is cancelled
|
||||
@@ -3049,7 +3026,6 @@ mod tests {
|
||||
skip_heal: Arc::new(AtomicBool::new(false)),
|
||||
local_disk: disk,
|
||||
pending_heals_changed: false,
|
||||
list_path_raw_options_observer: None,
|
||||
};
|
||||
|
||||
(scanner, temp_dir)
|
||||
@@ -4234,8 +4210,6 @@ mod tests {
|
||||
scanner.heal_object_select = 1;
|
||||
scanner.disks = disks;
|
||||
scanner.disks_quorum = 2;
|
||||
let (options_tx, mut options_rx) = mpsc::unbounded_channel();
|
||||
scanner.list_path_raw_options_observer = Some(options_tx);
|
||||
scanner.old_cache.replace(
|
||||
&format!("{bucket}/{object}"),
|
||||
bucket,
|
||||
@@ -4257,11 +4231,6 @@ mod tests {
|
||||
.expect("scan_folder should not hang after list_path_raw finishes")
|
||||
.expect("scan_folder should finish successfully");
|
||||
|
||||
let observed_options = tokio::time::timeout(Duration::from_secs(1), options_rx.recv())
|
||||
.await
|
||||
.expect("abandoned-child listing options should be observed promptly")
|
||||
.expect("abandoned-child listing options channel should remain open");
|
||||
assert_eq!(observed_options, (true, None, Some(SCANNER_LIST_PATH_RAW_STALL_TIMEOUT)));
|
||||
let root = scanner
|
||||
.new_cache
|
||||
.checked_flatten(bucket)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user