mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-19 14:32:28 +00:00
Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 66d727279b | |||
| a8b4367c5f | |||
| b9d3aea1a5 | |||
| f034e3701b | |||
| 55be4e3f79 | |||
| 5be4ef65e8 | |||
| 67e29c2143 | |||
| d6677ac005 | |||
| 24f3e783c2 | |||
| 583df55d90 | |||
| 670ae01f7a | |||
| 3a1f0c09b3 | |||
| eed50fc5ff | |||
| e462fafb65 | |||
| 9c0bc265c3 | |||
| 74bf3ca5fa | |||
| 31c4f4a7ff | |||
| 3fb8d861f0 | |||
| 4b386794e1 | |||
| 23a2593aac | |||
| 7b1befa6f9 | |||
| 2f3359df69 | |||
| ea522c18b6 | |||
| 9f13eb22a5 | |||
| fec2d4bb4b | |||
| 40f4e5e1e2 | |||
| 42a1417ed0 | |||
| a3d6b59d03 | |||
| 924eb06968 | |||
| 5a44944095 |
@@ -103,8 +103,6 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_dual_stack.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_mint_stack_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_conformance_rulings.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/compat_matrix_report.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
|
||||
@@ -723,9 +723,8 @@ test-group = 'ecstore-serial-flaky'
|
||||
# fixture as the existing smoke modules.
|
||||
#
|
||||
# Dual-stack differential (rustfs/backlog#2740): `dual_stack_differential::`
|
||||
# starts two single-node servers from the same binary, one per
|
||||
# RUSTFS_S3_STACK value, on random ports with isolated temp dirs, and runs one
|
||||
# SDK script covering every operation of `impl S3 for FS` against both. Its
|
||||
# starts two single-node servers on random ports with isolated temp
|
||||
# dirs, and runs one SDK script covering every S3 operation against both. Its
|
||||
# comparator unit tests need no server. It meets the admission criteria
|
||||
# unchanged; the module clause admits a new differential test without another
|
||||
# filter edit.
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
# Adding a target is one more [[targets]] table, for example:
|
||||
#
|
||||
# [[targets]]
|
||||
# name = "candidate-gateway"
|
||||
# name = "candidate-tuned"
|
||||
# binary = "candidate"
|
||||
# env = { RUSTFS_S3_STACK = "gateway" }
|
||||
# env = { RUSTFS_EXAMPLE_SETTING = "on" }
|
||||
|
||||
schema = 1
|
||||
|
||||
|
||||
@@ -28,8 +28,6 @@ rustfs/src/app/storage_api.rs
|
||||
rustfs/src/admin/router.rs
|
||||
rustfs/src/app/metadata_route.rs
|
||||
rustfs/src/server/http.rs
|
||||
rustfs/src/server/layer.rs
|
||||
rustfs/src/server/hybrid.rs
|
||||
rustfs/src/protocols/client.rs
|
||||
|
||||
# e2e oracle: the test target that verifies S3 behaviour against s3s
|
||||
|
||||
@@ -13,6 +13,6 @@
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=63
|
||||
s3_error_lines=209
|
||||
files=50
|
||||
s3_error_lines=103
|
||||
ecstore_files=0
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack leg
|
||||
description: Prove which S3 stack one e2e leg's servers selected and upload the leg for the summary (call with if always)
|
||||
|
||||
inputs:
|
||||
profile:
|
||||
description: The nextest profile the leg ran
|
||||
required: true
|
||||
stack:
|
||||
description: The stack the leg requested, legacy or gateway
|
||||
required: true
|
||||
logs:
|
||||
description: Directory or file holding the servers' output
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Prove which stack the servers selected
|
||||
shell: bash
|
||||
env:
|
||||
PROFILE: ${{ inputs.profile }}
|
||||
STACK: ${{ inputs.stack }}
|
||||
LOGS: ${{ inputs.logs }}
|
||||
run: |
|
||||
# Outside the workspace: e2e_binary.py refuses a run whose source tree changed.
|
||||
mkdir -p "$RUNNER_TEMP/leg"
|
||||
junit="target/nextest/${PROFILE}/junit.xml"
|
||||
if [ -s "$junit" ]; then cp "$junit" "$RUNNER_TEMP/leg/junit.xml"; fi
|
||||
# A gateway leg stopped by its preflight has no output; the preflight's stack.json stays.
|
||||
if [ -e "$LOGS" ]; then python3 scripts/dual_stack.py stack --expect "$STACK" --out "$RUNNER_TEMP/leg/stack.json" "$LOGS"; fi
|
||||
|
||||
- name: Upload the leg
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-${{ inputs.profile }}-${{ inputs.stack }}
|
||||
overwrite: true
|
||||
path: ${{ runner.temp }}/leg
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
@@ -1,56 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Both-stack summary
|
||||
description: Diff the legacy and gateway legs of this run (scripts/junit_diff.py) and upload one dated summary
|
||||
|
||||
inputs:
|
||||
profiles:
|
||||
description: Comma-separated nextest profiles whose two legs this run uploaded
|
||||
required: true
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Download the legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: dual-stack-leg-*
|
||||
path: legs
|
||||
|
||||
- name: Diff every profile and write the summary
|
||||
id: summary
|
||||
shell: bash
|
||||
env:
|
||||
PROFILES: ${{ inputs.profiles }}
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
date="$(date -u +%F)"
|
||||
echo "date=${date}" >> "$GITHUB_OUTPUT"
|
||||
mkdir -p legs
|
||||
python3 scripts/dual_stack.py summary --profiles "$PROFILES" --legs legs --date "$date" --out-dir dual-stack
|
||||
status=$?
|
||||
if [ -f dual-stack/summary.md ]; then cat dual-stack/summary.md >> "$GITHUB_STEP_SUMMARY"; fi
|
||||
exit "$status"
|
||||
|
||||
- name: Upload the dated summary
|
||||
if: always() && steps.summary.outputs.date != ''
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-summary-${{ steps.summary.outputs.date }}
|
||||
overwrite: true
|
||||
path: dual-stack
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
@@ -814,7 +814,7 @@
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "dual-stack-nightly.yml",
|
||||
"disposition": "both-stack JUnit diff of the two profiles only ci.yml runs; the gateway legs are report-only until T2.10",
|
||||
"disposition": "nightly run of the two e2e profiles only ci.yml runs, on the gateway stack, with the JUnit reports retained for 30 days",
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "added"
|
||||
},
|
||||
|
||||
+1
-173
@@ -92,7 +92,6 @@ jobs:
|
||||
connect: ${{ steps.scope.outputs.connect || 'true' }}
|
||||
packages: ${{ steps.scope.outputs.packages }}
|
||||
full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
|
||||
gateway_smoke: ${{ steps.scope.outputs.gateway_smoke || 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
@@ -119,7 +118,7 @@ jobs:
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=8$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'gateway_smoke=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
- name: Retain verification plan
|
||||
@@ -676,54 +675,6 @@ jobs:
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
test-and-lint-gateway-stack:
|
||||
name: Test and Lint (gateway-stack)
|
||||
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
|
||||
# `gateway-stack` feature, which no other pull-request lane lints or tests.
|
||||
# Unlike the dormant feature lanes above, this one runs for every code
|
||||
# change: each operation slice puts its handler tests and its refusal-matrix
|
||||
# row behind the feature, and a lane that ran only on demand would let them
|
||||
# rot behind a green pull request.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
|
||||
# build that reads the ci-dev cache written on this image.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev-ea7eec74fe14
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# A feature that does not compile fails here; nothing below is skipped
|
||||
# into a pass.
|
||||
- name: Run clippy with gateway-stack
|
||||
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
|
||||
|
||||
# The gateway stack's own tests: the assembly, the operation table and
|
||||
# its completeness, the shared handler sequence, the stack selection, and
|
||||
# the embedded server driven through the whole middleware stack. The
|
||||
# filter names modules, never operations.
|
||||
- name: Run tests with gateway-stack
|
||||
env:
|
||||
# The same bounded link concurrency as the other feature lanes.
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
run: |
|
||||
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
|
||||
--status-level all --final-status-level all \
|
||||
-E 'test(/gateway_stack|gateway_handlers|server::stack::/) | binary(embedded_gateway_stack_test)'
|
||||
|
||||
build-rustfs-debug-binary:
|
||||
name: Build RustFS Debug Binary
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
@@ -986,8 +937,6 @@ jobs:
|
||||
# Retain the producer image and fixture paths until the runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
RUSTFS_S3_STACK: legacy
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -1108,125 +1057,6 @@ jobs:
|
||||
path: ${{ runner.temp }}/rustfs-e2e-*/rustfs.log
|
||||
retention-days: 3
|
||||
|
||||
# The same e2e-smoke suite against the same debug binary with the gateway
|
||||
# S3 stack selected (rustfs/backlog#2734). Report-only: it is outside the
|
||||
# Test and Lint aggregate and becomes required at T2.10. The test-binary
|
||||
# download, identity, membership, run and completeness steps must stay
|
||||
# identical to e2e-tests; scripts/ci_gate.py enforces that.
|
||||
#
|
||||
# When it runs: every push to main or gateway/integration, merge group,
|
||||
# nightly schedule and manual dispatch, and every pull request that changes a
|
||||
# gateway-stack input (GATEWAY_SMOKE_PATHS and the rustfs-gateway pins in
|
||||
# scripts/ci_gate.py, published by Select CI scope as gateway_smoke).
|
||||
# Why it is skipped on every other pull request: until T2.10 the gateway
|
||||
# stack does not serve the whole smoke suite, so the lane fails on any
|
||||
# revision. A pull request that touches none of its inputs can neither cause
|
||||
# nor fix that failure, and a red check there says nothing about the change.
|
||||
# Skipped is the honest state; a failure must never be rendered green, so no
|
||||
# step here may carry continue-on-error. A selection that is missing or
|
||||
# unreadable runs the lane. At T2.10, when the lane becomes required, it
|
||||
# must run on every pull request again: delete the selection, do not extend
|
||||
# it.
|
||||
e2e-tests-gateway:
|
||||
name: e2e-smoke (gateway, report-only until T2.10)
|
||||
needs: [ classify-changes, build-rustfs-debug-binary ]
|
||||
if: github.event_name != 'pull_request' || needs.classify-changes.outputs.gateway_smoke != 'false'
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 30
|
||||
continue-on-error: true
|
||||
env:
|
||||
RUSTFS_S3_STACK: gateway
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install verification runtime
|
||||
uses: ./.github/actions/test-runtime
|
||||
|
||||
- name: Download E2E test binaries
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: e2e-test-binaries
|
||||
path: ${{ runner.temp }}
|
||||
|
||||
- name: Verify E2E archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install awscurl
|
||||
run: |
|
||||
python3 -m pip install --user --upgrade pip "awscurl==0.44"
|
||||
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Verify awscurl
|
||||
run: test -x "$AWSCURL_PATH"
|
||||
|
||||
# Download after the cache restore so the freshly built binary from the
|
||||
# build job always wins over anything restored into target/debug.
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
# Build the e2e test graph once. The archive is reused by the smoke
|
||||
# selection guard, security exact-count check, and run below, avoiding a
|
||||
# second compile of the same e2e_test target on cold runners (backlog#1645).
|
||||
- name: Verify e2e smoke test membership
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
run: |
|
||||
cargo nextest list --profile e2e-smoke --archive-file "${NEXTEST_ARCHIVE}" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json > "${NEXTEST_LISTING}"
|
||||
python3 ./scripts/check_test_wiring.py --check-profile e2e-smoke "${NEXTEST_LISTING}"
|
||||
./scripts/check_security_smoke_count.sh check "${NEXTEST_LISTING}"
|
||||
|
||||
# PR smoke subset of the in-repo e2e suite (backlog#1149 ci-4). The
|
||||
# profile.e2e-smoke default-filter in .config/nextest.toml is the single
|
||||
# wiring mechanism for e2e tests in CI — extend that filter instead of
|
||||
# adding new e2e jobs here. Each test spawns its own rustfs server on a
|
||||
# random port and reuses the downloaded debug binary above.
|
||||
- name: Run e2e smoke suite
|
||||
env:
|
||||
NEXTEST_ARCHIVE: ${{ runner.temp }}/e2e-tests.tar.zst
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-smoke-logs
|
||||
SAMPLER_INTERVAL_SECS: "15"
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start e2e-smoke
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json --workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all --failure-output final
|
||||
|
||||
- name: Verify e2e smoke execution completeness
|
||||
run: python3 scripts/ci_test_artifact.py evidence "$RUNNER_TEMP/rustfs-e2e-smoke-list.json" target/nextest/e2e-smoke/junit.xml
|
||||
|
||||
- name: Upload e2e smoke diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-smoke-gateway-junit-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: target/nextest/e2e-smoke/junit.xml
|
||||
if-no-files-found: warn
|
||||
|
||||
e2e-full:
|
||||
name: End-to-End Tests (full merge gate)
|
||||
# Merge gate only (backlog#1149 ci-5): the never-automated user-visible
|
||||
@@ -1612,7 +1442,6 @@ jobs:
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
@@ -1648,7 +1477,6 @@ jobs:
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- test-and-lint-gateway-stack
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
|
||||
@@ -12,21 +12,18 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Nightly both-stack diff of the two e2e profiles only ci.yml runs, e2e-smoke
|
||||
# and e2e-full (rustfs/backlog#2772, T3.1). The other profiles carry a `stack`
|
||||
# dimension in their own workflows, where the existing lane is the legacy leg.
|
||||
# Nightly run of the two e2e profiles only ci.yml runs, e2e-smoke and e2e-full
|
||||
# (rustfs/backlog#2772, T3.1), on the gateway stack, which is the only S3 stack.
|
||||
# The workflow keeps its file and registry name from the period when it diffed two
|
||||
# stacks. Each leg uploads its JUnit report and server logs for 30 days; the one
|
||||
# comparison against the last legacy record (the `dual-stack-leg-<profile>-legacy`
|
||||
# artifacts of the final both-stack run) is made by hand with scripts/junit_diff.py.
|
||||
#
|
||||
# One build serves every leg. One preflight starts that binary with
|
||||
# RUSTFS_S3_STACK=gateway (scripts/dual_stack.py preflight); without a proven
|
||||
# gateway start the gateway legs do not run and the summary says `skipped:
|
||||
# gateway stack refuses startup: <why>`. Which stack a leg ran is read from the
|
||||
# servers' own start-up line. The gateway legs are report-only until T2.10.
|
||||
#
|
||||
# Left out of both stacks: `dual_stack_differential::` (servers with an explicit
|
||||
# Left out: `dual_stack_differential::` (servers with an explicit
|
||||
# stack), `fake_s3_target::`, the upgrade write-readiness cases and one deadline
|
||||
# case (no server), and `distributed_startup_regression_test::` (needs ci.yml's
|
||||
# hooks-binary manifest). Both e2e-full stacks use ci.yml's pinned mc and
|
||||
# Vault binaries and network fault-injection tools.
|
||||
# hooks-binary manifest). e2e-full uses ci.yml's pinned mc and Vault binaries
|
||||
# and network fault-injection tools.
|
||||
|
||||
name: dual-stack-nightly
|
||||
|
||||
@@ -50,13 +47,11 @@ defaults:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build once and probe the gateway stack
|
||||
name: Build once
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# The runner and cache of ci.yml's build-rustfs-debug-binary.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
outputs:
|
||||
stacks: ${{ steps.preflight.outputs.stacks }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -86,47 +81,15 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Without a proven gateway start there is no gateway leg to run.
|
||||
- name: Gateway preflight
|
||||
id: preflight
|
||||
run: |
|
||||
if python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/preflight/stack.json"; then
|
||||
echo 'stacks=["legacy","gateway"]' >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo 'stacks=["legacy"]' >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# The evidence stands in for a gateway leg that is skipped; a leg that runs overwrites it.
|
||||
- name: Upload the preflight evidence for e2e-smoke
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-smoke-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload the preflight evidence for e2e-full
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: dual-stack-leg-e2e-full-gateway
|
||||
path: ${{ runner.temp }}/preflight
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
leg:
|
||||
name: ${{ matrix.profile }} (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
name: ${{ matrix.profile }}
|
||||
needs: [ build ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 120
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
profile: [ e2e-smoke, e2e-full ]
|
||||
stack: ${{ fromJSON(needs.build.outputs.stacks) }}
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -208,10 +171,9 @@ jobs:
|
||||
run: |
|
||||
"$RUSTFS_TEST_VAULT_BIN" version
|
||||
|
||||
- name: Run the profile on the ${{ matrix.stack }} stack
|
||||
- name: Run the profile
|
||||
env:
|
||||
PROFILE: ${{ matrix.profile }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/server-logs
|
||||
run: |
|
||||
chmod +x target/debug/rustfs
|
||||
@@ -220,23 +182,14 @@ jobs:
|
||||
filter="$filter & not test(=list_objects_v2_pagination_test::tests::test_list_objects_v2_deadline_rejects_unresponsive_peer)"
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile "$PROFILE" -p e2e_test -E "$filter"
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
- name: Upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with: { profile: "${{ matrix.profile }}", stack: "${{ matrix.stack }}", logs: "${{ runner.temp }}/server-logs" }
|
||||
|
||||
summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [ build, leg ]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-smoke,e2e-full
|
||||
name: e2e-nightly-${{ matrix.profile }}
|
||||
overwrite: true
|
||||
path: |
|
||||
target/nextest/${{ matrix.profile }}/junit.xml
|
||||
${{ runner.temp }}/server-logs
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
@@ -53,14 +53,11 @@ concurrency:
|
||||
jobs:
|
||||
distributed:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Distributed 4-node 4-disk e2e (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
name: Distributed 4-node 4-disk e2e
|
||||
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
|
||||
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
@@ -132,11 +129,6 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify distributed e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -148,7 +140,6 @@ jobs:
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
FILTER: ${{ inputs.filter }}
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -f target/nextest/e2e-distributed/junit.xml
|
||||
@@ -159,19 +150,11 @@ jobs:
|
||||
fi
|
||||
test -s target/nextest/e2e-distributed/junit.xml
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-distributed
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-distributed-logs
|
||||
|
||||
- name: Upload distributed e2e diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-distributed-${{ github.run_number }}-${{ matrix.stack }}
|
||||
name: e2e-distributed-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-distributed/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-distributed-list.json
|
||||
@@ -191,20 +174,6 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [distributed]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-distributed
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [distributed]
|
||||
|
||||
@@ -54,18 +54,15 @@ concurrency:
|
||||
jobs:
|
||||
repl-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Replication e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
name: Replication e2e (nightly)
|
||||
# dual-node tests spawn two full rustfs servers each; the extra headroom
|
||||
# over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out
|
||||
# from starving the runner. Mirrors the ILM serial lane's runner class.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 45
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ${{ matrix.stack == 'gateway' && 'gateway-stack' || '' }}
|
||||
RUSTFS_BUILD_FEATURES: ""
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -99,11 +96,6 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features "$RUSTFS_BUILD_FEATURES"
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify replication e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
@@ -114,22 +106,13 @@ jobs:
|
||||
- name: Run replication e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-repl-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
|
||||
|
||||
- name: Upload nextest junit report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}-${{ matrix.stack }}
|
||||
name: e2e-replication-nightly-junit-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-repl-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
|
||||
@@ -139,12 +122,9 @@ jobs:
|
||||
|
||||
cluster-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Cluster fault e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
name: Cluster fault e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -165,11 +145,6 @@ jobs:
|
||||
run: |
|
||||
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
- name: Verify cluster fault e2e membership
|
||||
env:
|
||||
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -181,22 +156,13 @@ jobs:
|
||||
env:
|
||||
CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-nightly
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-nightly-logs
|
||||
|
||||
- name: Upload cluster fault diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
name: e2e-cluster-nightly-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-nightly/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-nightly-list.json
|
||||
@@ -206,15 +172,12 @@ jobs:
|
||||
|
||||
protocols-nightly:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
name: Protocol e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
name: Protocol e2e (nightly)
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTFS_BUILD_FEATURES: ${{ matrix.stack == 'gateway' && 'ftps,webdav,sftp,gateway-stack' || 'ftps,webdav,sftp' }}
|
||||
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -238,11 +201,6 @@ jobs:
|
||||
- name: Build protocol server
|
||||
run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
||||
- name: Verify protocol e2e membership
|
||||
env:
|
||||
@@ -254,28 +212,16 @@ jobs:
|
||||
- name: Run protocol e2e nightly suite
|
||||
env:
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
# These servers set no log level of their own; the stack proof reads their start-up line.
|
||||
RUST_LOG: rustfs=info
|
||||
# The servers inherit nextest's stdout (--no-capture): keep it where the stack proof reads it.
|
||||
run: |
|
||||
set -o pipefail
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture 2>&1 | tee "$RUSTFS_E2E_LOG_DIR/nextest.log"
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-protocols
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-protocol-e2e-logs
|
||||
|
||||
- name: Upload protocol diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}-${{ matrix.stack }}
|
||||
name: e2e-protocol-nightly-${{ github.run_number }}
|
||||
path: |
|
||||
target/nextest/e2e-protocols/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-protocols-list.json
|
||||
@@ -283,20 +229,6 @@ jobs:
|
||||
retention-days: 7
|
||||
if-no-files-found: warn
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-repl-nightly,e2e-nightly,e2e-protocols
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [repl-nightly, cluster-nightly, protocols-nightly]
|
||||
|
||||
@@ -15,31 +15,20 @@
|
||||
# Gateway black-box acceptance lanes (rustfs/backlog#2775 T3.4, #2776 T3.5).
|
||||
#
|
||||
# Runs two suites of the rustfs/gateway repository against the candidate binary,
|
||||
# once per S3 stack (RUSTFS_S3_STACK=legacy and =gateway), one server per stack:
|
||||
# one server:
|
||||
#
|
||||
# - the conformance corpus, `--profile rustfs`. That profile describes the
|
||||
# gateway edge, so scripts/check_conformance_rulings.py judges the gateway
|
||||
# stack against .config/conformance-rulings.toml (red: an unruled failure, a
|
||||
# ruled case failing differently, a stale ruling, a skip on the server's own
|
||||
# answer) and publishes the legacy run as a reference that is never judged:
|
||||
# it only has to serve, stay up and complete;
|
||||
# - the client matrix, ci/compat/run_matrix.sh --external, judged per stack by
|
||||
# - the conformance corpus, `--profile rustfs`, judged by
|
||||
# scripts/check_conformance_rulings.py against .config/conformance-rulings.toml
|
||||
# (red: an unruled failure, a ruled case failing differently, a stale ruling,
|
||||
# a skip on the server's own answer);
|
||||
# - the client matrix, ci/compat/run_matrix.sh --external, judged by
|
||||
# scripts/compat_matrix_report.py (no `fail`, no `sut-unregistered`, every
|
||||
# client ran; a client that could not be installed is reported as skipped).
|
||||
# The legacy matrix is the baseline: every cell that passes on legacy must
|
||||
# pass on gateway (`compare`).
|
||||
#
|
||||
# The suites are checked out at the rustfs/gateway rev Cargo.toml pins, so the
|
||||
# suite and the framework under test never drift apart. The verdict of each
|
||||
# stack is computed from what the run left on disk (startup record, the stack
|
||||
# the server's own log names, runner exit code, liveness after the corpus,
|
||||
# report), never from the exit status of an earlier step. Needs
|
||||
# scripts/dual_stack.py and the `s3_stack` startup field of rustfs/rustfs#9367.
|
||||
#
|
||||
# The gateway stack is report-only until T2.10: its job may fail without failing
|
||||
# the workflow, and its summary says what it observed (did not start, or how many
|
||||
# cases ran, passed, failed and were answered 501). The stack comparison is
|
||||
# report-only for the same reason; its conformance half is information only.
|
||||
# suite and the framework under test never drift apart. The verdict is computed
|
||||
# from what the run left on disk (startup record, runner exit code, liveness
|
||||
# after the corpus, report), never from the exit status of an earlier step.
|
||||
#
|
||||
# Not a pull-request gate. Scheduled failures are owned by the scheduled-validation
|
||||
# watchdog (.github/scheduled-validations.json).
|
||||
@@ -166,21 +155,15 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
acceptance:
|
||||
name: ${{ matrix.stack == 'gateway' && 'Acceptance (gateway, report-only until T2.10)' || 'Acceptance (legacy)' }}
|
||||
name: Acceptance
|
||||
needs: [ plan, build-candidate, build-gateway-tools ]
|
||||
# GitHub-hosted like the s3-tests targets of reference-parity.yml: the job
|
||||
# only runs prebuilt binaries and the pinned clients, none of which needs Docker.
|
||||
runs-on: ubuntu-latest
|
||||
# About 3x the 15 minutes the legacy job took in run 43325101234 (client install about
|
||||
# About 3x the 15 minutes the job took in run 43325101234 (client install about
|
||||
# 8, matrix about 3, corpus about 3), so a hung server costs 45 runner minutes, not more.
|
||||
timeout-minutes: 45
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
stack: [ legacy, gateway ]
|
||||
env:
|
||||
STACK: ${{ matrix.stack }}
|
||||
ENDPOINT: http://127.0.0.1:9000
|
||||
# The conformance suite signs with these fixed fixture credentials (the AWS
|
||||
# documentation example pair), so the candidate is started with them and the
|
||||
@@ -213,10 +196,8 @@ jobs:
|
||||
name: gateway-acceptance-tools
|
||||
path: tools
|
||||
|
||||
# Records `ready`, or what the process said when it exited, in lane/startup.txt. `ready`
|
||||
# also needs the server's own `server_starting` log event to name the requested stack
|
||||
# (its `s3_stack` field comes from rustfs/rustfs#9367; scripts/dual_stack.py reads it).
|
||||
- name: Start candidate (${{ matrix.stack }} stack)
|
||||
# Records `ready`, or what the process said when it exited, in lane/startup.txt.
|
||||
- name: Start candidate
|
||||
run: |
|
||||
set -uo pipefail
|
||||
chmod +x target/debug/rustfs tools/*
|
||||
@@ -224,7 +205,7 @@ jobs:
|
||||
target/debug/rustfs --version | tee lane/candidate-version.txt
|
||||
# The four directories share the runner's one disk; the bypass is the CI case the
|
||||
# physical-disk guard sanctions (as in mint.yml).
|
||||
RUSTFS_S3_STACK="$STACK" RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true RUST_LOG=info target/debug/rustfs --address 127.0.0.1:9000 \
|
||||
RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true RUST_LOG=info target/debug/rustfs --address 127.0.0.1:9000 \
|
||||
--access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY" "$RUNNER_TEMP"/data/rustfs{0..3} \
|
||||
> lane/rustfs.log 2>&1 &
|
||||
pid=$!
|
||||
@@ -239,13 +220,8 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
if curl -sf "$ENDPOINT/health/ready" >/dev/null; then
|
||||
if python3 scripts/dual_stack.py stack --expect "$STACK" --out lane/stack.json lane/rustfs.log > /dev/null; then
|
||||
echo ready > lane/startup.txt
|
||||
exit 0
|
||||
fi
|
||||
echo "did not start: stack not observed" > lane/startup.txt
|
||||
cat lane/startup.txt
|
||||
exit 1
|
||||
echo ready > lane/startup.txt
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
@@ -338,58 +314,23 @@ jobs:
|
||||
run: |
|
||||
set -uo pipefail
|
||||
status=0
|
||||
python3 scripts/check_conformance_rulings.py --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/compat_matrix_report.py check --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/check_conformance_rulings.py --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
python3 scripts/compat_matrix_report.py check --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
|
||||
exit "$status"
|
||||
|
||||
- name: Upload lane results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: gateway-acceptance-lane-${{ matrix.stack }}
|
||||
name: gateway-acceptance-lane
|
||||
overwrite: true
|
||||
path: |
|
||||
lane/*.txt
|
||||
lane/*.log
|
||||
lane/report.json
|
||||
lane/stack.json
|
||||
lane/compat/matrix.json
|
||||
lane/compat/probe.jsonl
|
||||
lane/compat/sut.log
|
||||
lane/compat/results/**
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
compare:
|
||||
name: Compare stacks (report-only until T2.10)
|
||||
needs: [ acceptance ]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download lane results
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: gateway-acceptance-lane-*
|
||||
path: lanes
|
||||
|
||||
- name: Diff legacy and gateway
|
||||
run: |
|
||||
set -uo pipefail
|
||||
status=0
|
||||
{
|
||||
python3 scripts/check_conformance_rulings.py \
|
||||
--diff lanes/gateway-acceptance-lane-legacy lanes/gateway-acceptance-lane-gateway || status=1
|
||||
echo
|
||||
python3 scripts/compat_matrix_report.py diff \
|
||||
--legacy lanes/gateway-acceptance-lane-legacy --gateway lanes/gateway-acceptance-lane-gateway || status=1
|
||||
} > diff.md
|
||||
cat diff.md
|
||||
cat diff.md >> "$GITHUB_STEP_SUMMARY"
|
||||
exit "$status"
|
||||
|
||||
@@ -44,13 +44,6 @@
|
||||
# well inside the 120-minute budget and buys deterministic infra. The
|
||||
# docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
|
||||
# has fewer cores and reintroduces fleet-state risk for no reliability gain.
|
||||
#
|
||||
# Stack dimension (rustfs/backlog#2774, T3.3): one image, run with RUSTFS_S3_STACK=legacy and
|
||||
# =gateway. `compare` (scripts/mint_stack_diff.py) holds the gate that every case legacy passes
|
||||
# passes on gateway with no new failure. The gateway leg and `compare` are report-only until
|
||||
# T2.10. A gateway server that refuses to start stops its leg at "Wait for RustFS ready", before
|
||||
# mint runs; the summary says `skipped: gateway stack refuses startup: <why>`. The stack is read
|
||||
# from the server's own start-up line (scripts/dual_stack.py stack).
|
||||
|
||||
name: mint
|
||||
|
||||
@@ -145,16 +138,8 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
mint:
|
||||
name: mint (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
name: mint
|
||||
needs: [image]
|
||||
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
stack: [legacy, gateway]
|
||||
env:
|
||||
STACK: ${{ matrix.stack }}
|
||||
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
|
||||
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
|
||||
# label was abandoned (no docker.sock on the pod its only run landed on).
|
||||
@@ -194,7 +179,6 @@ jobs:
|
||||
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
|
||||
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
|
||||
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
|
||||
-e RUSTFS_S3_STACK="${STACK}" \
|
||||
-e RUST_LOG=rustfs=info \
|
||||
-v /tmp/rustfs-mint:/data \
|
||||
rustfs-ci
|
||||
@@ -291,47 +275,13 @@ jobs:
|
||||
mkdir -p artifacts/rustfs-mint
|
||||
docker logs rustfs-mint > artifacts/rustfs-mint/rustfs.log 2>&1 || true
|
||||
|
||||
- name: Prove which stack the server selected
|
||||
if: always()
|
||||
run: python3 scripts/dual_stack.py stack --expect "${STACK}" --out artifacts/stack.json artifacts/rustfs-mint
|
||||
|
||||
- name: Upload artifacts
|
||||
if: always() && env.ACT != 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: mint-${{ matrix.stack }}
|
||||
name: mint
|
||||
path: artifacts/**
|
||||
|
||||
compare:
|
||||
name: Compare stacks (report-only until T2.10)
|
||||
needs: [mint]
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download both legs
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: mint-{legacy,gateway}
|
||||
path: legs
|
||||
|
||||
# A leg that never uploaded leaves no directory: create both so that the
|
||||
# comparison reports "no result" for it instead of failing to find it.
|
||||
- name: Compare the stacks
|
||||
run: |
|
||||
# The shell runs with -e; the exit status is reported after the summary is written.
|
||||
set +e -u -o pipefail
|
||||
mkdir -p legs/mint-legacy legs/mint-gateway
|
||||
python3 scripts/mint_stack_diff.py legs/mint-legacy legs/mint-gateway | tee report.txt
|
||||
status="${PIPESTATUS[0]}"
|
||||
{ echo '## Mint: gateway versus legacy'; echo; echo '```'; cat report.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
|
||||
exit "$status"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [image, mint]
|
||||
|
||||
@@ -68,14 +68,11 @@ env:
|
||||
|
||||
jobs:
|
||||
minio-source:
|
||||
name: MinIO source (read-through, list-through, backfill, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
|
||||
name: MinIO source (read-through, list-through, backfill)
|
||||
# Keep this scheduled interoperability lane scoped to the upstream repo.
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
# The gateway leg is report-only until T2.10.
|
||||
continue-on-error: ${{ matrix.stack == 'gateway' }}
|
||||
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost
|
||||
# Fixed credentials of the container this job starts and throws away;
|
||||
@@ -140,11 +137,6 @@ jobs:
|
||||
- name: Build the RustFS binary under test
|
||||
run: python3 scripts/e2e_binary.py build --bins
|
||||
|
||||
# A gateway binary that refuses to start skips the suite; the summary says why.
|
||||
- name: Gateway preflight
|
||||
if: matrix.stack == 'gateway'
|
||||
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
|
||||
|
||||
# The lane selects tests by module, so a rename would quietly shrink it.
|
||||
# The committed digest in .config/e2e-odm-interop-selection.txt fails
|
||||
# closed on that.
|
||||
@@ -155,18 +147,9 @@ jobs:
|
||||
|
||||
- name: Run the interop cases against MinIO
|
||||
env:
|
||||
RUSTFS_S3_STACK: ${{ matrix.stack }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
|
||||
|
||||
- name: Prove the stack and upload the leg
|
||||
if: always()
|
||||
uses: ./.github/actions/dual-stack-leg
|
||||
with:
|
||||
profile: e2e-odm-interop
|
||||
stack: ${{ matrix.stack }}
|
||||
logs: ${{ runner.temp }}/rustfs-e2e-logs
|
||||
|
||||
- name: Build the MinIO interop report
|
||||
if: always()
|
||||
uses: ./.github/actions/odm-interop-report
|
||||
@@ -190,7 +173,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.stack }}
|
||||
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
artifacts/odm-interop/minio
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
@@ -316,20 +299,6 @@ jobs:
|
||||
target/nextest/e2e-odm-interop/junit.xml
|
||||
retention-days: 14
|
||||
|
||||
dual-stack-summary:
|
||||
name: Both-stack diff summary
|
||||
needs: [minio-source]
|
||||
if: github.repository == 'rustfs/rustfs' && always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/dual-stack-summary
|
||||
with:
|
||||
profiles: e2e-odm-interop
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [minio-source, cloud-source]
|
||||
|
||||
@@ -32,9 +32,9 @@
|
||||
#
|
||||
# The `differential` job adds the 73-operation dual-stack differential
|
||||
# (crates/e2e_test/src/dual_stack_differential, rustfs/backlog#2740) against the
|
||||
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY,
|
||||
# started without RUSTFS_S3_STACK), side B is the candidate built above on its
|
||||
# legacy stack. It reuses the candidate build artifact, never builds the candidate
|
||||
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY),
|
||||
# side B is the candidate built above.
|
||||
# It reuses the candidate build artifact, never builds the candidate
|
||||
# twice, and prints reference_side, version_a and version_b from the report the test
|
||||
# wrote; it fails when side A was not the reference or both sides report the same
|
||||
# version. Every (op, field, a, b) difference goes to the job summary. Report-only
|
||||
@@ -310,7 +310,7 @@ jobs:
|
||||
chmod +x target/debug/rustfs
|
||||
test -x "$REFERENCE_PATH"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
# Side A: the released binary. Side B: the candidate on its legacy stack.
|
||||
# Side A: the released binary. Side B: the candidate.
|
||||
export RUSTFS_E2E_REFERENCE_BINARY="$REFERENCE_PATH"
|
||||
test_name='dual_stack_differential::tests::both_stacks_answer_every_operation_identically'
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
|
||||
|
||||
@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Deployment
|
||||
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
|
||||
|
||||
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
|
||||
|
||||
- Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).
|
||||
|
||||
Generated
+3
@@ -11377,7 +11377,9 @@ dependencies = [
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-utils",
|
||||
"rustix",
|
||||
"s3s",
|
||||
"same-file",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
@@ -11391,6 +11393,7 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -1235,23 +1235,66 @@ impl Serialize for DataUsageEntry {
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
self.serialize_fields(serializer, &self.children, &self.replication_stats, &self.all_tier_stats)
|
||||
}
|
||||
}
|
||||
|
||||
struct OrderedUsageEntry<'a>(&'a DataUsageEntry);
|
||||
|
||||
impl Serialize for OrderedUsageEntry<'_> {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut children: Vec<_> = self.0.children.iter().collect();
|
||||
children.sort_unstable();
|
||||
// MessagePack decoders accept struct fields as positional sequences.
|
||||
// Include the nested target and tier maps in stable order as well.
|
||||
let replication = self.0.replication_stats.as_ref().map(|stats| {
|
||||
let ReplicationAllStats {
|
||||
targets,
|
||||
replica_size,
|
||||
replica_count,
|
||||
} = stats;
|
||||
let targets: std::collections::BTreeMap<_, _> = targets.iter().collect();
|
||||
(targets, replica_size, replica_count)
|
||||
});
|
||||
let tiers = self.0.all_tier_stats.as_ref().map(|stats| {
|
||||
let AllTierStats { tiers } = stats;
|
||||
let tiers: std::collections::BTreeMap<_, _> = tiers.iter().collect();
|
||||
(tiers,)
|
||||
});
|
||||
self.0.serialize_fields(serializer, &children, &replication, &tiers)
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageEntry {
|
||||
/// Stable MessagePack encoding for independently reusable scanner checkpoint pages.
|
||||
pub fn ordered_encoding(&self) -> impl Serialize + '_ {
|
||||
OrderedUsageEntry(self)
|
||||
}
|
||||
|
||||
fn serialize_fields<S: serde::Serializer, C: Serialize, R: Serialize, T: Serialize>(
|
||||
&self,
|
||||
serializer: S,
|
||||
children: &C,
|
||||
replication: &R,
|
||||
tiers: &T,
|
||||
) -> Result<S::Ok, S::Error> {
|
||||
// Keep entries map-encoded so older readers can ignore fields appended
|
||||
// by newer scanner versions during rolling upgrades. The derived
|
||||
// (array) encoding made any appended field a decode error for them.
|
||||
let mut state = serializer.serialize_map(Some(
|
||||
11 + usize::from(self.unknown_tier_stats.is_some()) + usize::from(self.tier_accounting_proof.is_some()),
|
||||
))?;
|
||||
state.serialize_entry("children", &self.children)?;
|
||||
state.serialize_entry("children", children)?;
|
||||
state.serialize_entry("size", &self.size)?;
|
||||
state.serialize_entry("objects", &self.objects)?;
|
||||
state.serialize_entry("versions", &self.versions)?;
|
||||
state.serialize_entry("delete_markers", &self.delete_markers)?;
|
||||
state.serialize_entry("obj_sizes", &self.obj_sizes)?;
|
||||
state.serialize_entry("obj_versions", &self.obj_versions)?;
|
||||
state.serialize_entry("replication_stats", &self.replication_stats)?;
|
||||
state.serialize_entry("replication_stats", replication)?;
|
||||
state.serialize_entry("compacted", &self.compacted)?;
|
||||
state.serialize_entry("failed_objects", &self.failed_objects)?;
|
||||
state.serialize_entry("all_tier_stats", &self.all_tier_stats)?;
|
||||
state.serialize_entry("all_tier_stats", tiers)?;
|
||||
// Keep the legacy no-unknown shape byte-for-byte stable. Once unknown
|
||||
// accounting exists, append its map field before the optional proof.
|
||||
if let Some(unknown_tier_stats) = self.unknown_tier_stats.as_ref() {
|
||||
@@ -2534,6 +2577,32 @@ mod tests {
|
||||
entry
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordered_checkpoint_entry_round_trips_and_stays_identical_after_reload() {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
let mut replication = ReplicationAllStats {
|
||||
replica_size: 998,
|
||||
replica_count: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let mut tiers = AllTierStats::default();
|
||||
for index in 1..=32 {
|
||||
let key = format!("source-{index}");
|
||||
entry.children.insert(key.clone());
|
||||
replication.targets.insert(key.clone(), ReplicationTargetUsage::default());
|
||||
tiers.tiers.insert(key, TierStats::default());
|
||||
}
|
||||
entry.replication_stats = Some(replication);
|
||||
entry.all_tier_stats = Some(tiers);
|
||||
let bytes = rmp_serde::to_vec(&entry.ordered_encoding()).expect("encode ordered checkpoint");
|
||||
let reloaded: DataUsageEntry = rmp_serde::from_slice(&bytes).expect("read with the existing codec");
|
||||
assert_eq!(reloaded.children, entry.children);
|
||||
let replicas = reloaded.replication_stats.as_ref().expect("replication stats");
|
||||
assert_eq!((replicas.targets.len(), replicas.replica_size, replicas.replica_count), (32, 998, 3));
|
||||
assert_eq!(reloaded.all_tier_stats.as_ref().expect("tier stats").tiers.len(), 32);
|
||||
assert_eq!(rmp_serde::to_vec(&reloaded.ordered_encoding()).expect("encode reloaded entry"), bytes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_stats_survive_entry_merge() {
|
||||
let mut left = tier_entry(
|
||||
|
||||
@@ -171,70 +171,6 @@ impl DataUsageBucketObservation {
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
/// Hide values that cannot be bound to the current storage-owned identity.
|
||||
///
|
||||
/// `incarnations` contains canonical non-nil identities read from storage;
|
||||
/// a missing entry means identity was unavailable. Neither legacy values
|
||||
/// without identity nor metadata read failures establish confirmed zero.
|
||||
/// Changed responses are display-only and cannot authorize quota or writes.
|
||||
/// If retained counters overflow, all values remain unknown.
|
||||
/// Returns the number of canonical values removed.
|
||||
pub fn retain_current_bucket_usage_for_admin(&mut self, incarnations: &HashMap<String, String>) -> usize {
|
||||
let original_count = self.buckets_usage.len();
|
||||
let original_observation_count = self.usage_snapshot_bucket_observations.len();
|
||||
let matches_current = |bucket: &String, usage: &BucketUsageInfo| {
|
||||
incarnations.get(bucket).is_some_and(|current| {
|
||||
!current.is_empty()
|
||||
&& current != "00000000-0000-0000-0000-000000000000"
|
||||
&& usage.bucket_incarnation.as_ref() == Some(current)
|
||||
})
|
||||
};
|
||||
self.buckets_usage.retain(|bucket, usage| matches_current(bucket, usage));
|
||||
self.usage_snapshot_bucket_observations.retain(|bucket, observation| {
|
||||
matches_current(bucket, &observation.usage)
|
||||
&& observation
|
||||
.sources
|
||||
.iter()
|
||||
.all(|source| incarnations.get(bucket) == Some(&source.bucket_incarnation))
|
||||
});
|
||||
if original_count == self.buckets_usage.len()
|
||||
&& original_observation_count == self.usage_snapshot_bucket_observations.len()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
let totals = self.buckets_usage.values().try_fold([0_u64; 4], |total, usage| {
|
||||
Some([
|
||||
total[0].checked_add(usage.size)?,
|
||||
total[1].checked_add(usage.objects_count)?,
|
||||
total[2].checked_add(usage.versions_count)?,
|
||||
total[3].checked_add(usage.delete_markers_count)?,
|
||||
])
|
||||
});
|
||||
let [size, objects, versions, markers] = totals.unwrap_or_else(|| {
|
||||
self.buckets_usage.clear();
|
||||
self.usage_snapshot_bucket_observations.clear();
|
||||
[0; 4]
|
||||
});
|
||||
self.bucket_sizes = self
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
self.buckets_count = u64::try_from(self.buckets_usage.len()).unwrap_or(u64::MAX);
|
||||
self.objects_total_size = size;
|
||||
self.objects_total_count = objects;
|
||||
self.versions_total_count = versions;
|
||||
self.delete_markers_total_count = markers;
|
||||
self.replication_info.clear();
|
||||
self.tier_stats = None;
|
||||
self.unknown_tier_stats = None;
|
||||
self.usage_snapshot_complete = false;
|
||||
self.usage_snapshot_partial = true;
|
||||
self.usage_snapshot_converged = Some(false);
|
||||
self.usage_snapshot_admin_projection = true;
|
||||
original_count.saturating_sub(self.buckets_usage.len())
|
||||
}
|
||||
|
||||
/// Overlay validated companions on an owned admin response only.
|
||||
///
|
||||
/// `incarnations` must come from current storage metadata, not the companion.
|
||||
|
||||
@@ -321,110 +321,3 @@ fn provenance_and_companions_change_candidate_bytes_and_round_trip_as_maps() {
|
||||
let restored: DataUsageInfo = rmp_serde::from_slice(&named).expect("read named map with additive fields");
|
||||
assert_eq!(restored, canonical);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_identity_filter_preserves_an_unchanged_complete_snapshot() {
|
||||
let mut candidate = snapshot();
|
||||
let original = candidate.clone();
|
||||
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&incarnations()), 0);
|
||||
assert_eq!(candidate, original);
|
||||
assert!(candidate.is_complete_bucket_usage_snapshot());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_identity_filter_hides_unbound_values_without_confirming_zero() {
|
||||
for identity in [
|
||||
None,
|
||||
Some(""),
|
||||
Some("corrupt"),
|
||||
Some("00000000-0000-0000-0000-000000000000"),
|
||||
Some("22222222-2222-4222-8222-222222222222"),
|
||||
] {
|
||||
let mut candidate = snapshot();
|
||||
candidate.usage_snapshot_bucket_observations.clear();
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.bucket_incarnation = identity.map(str::to_string);
|
||||
assert_eq!(
|
||||
candidate.retain_current_bucket_usage_for_admin(&incarnations()),
|
||||
1,
|
||||
"identity {identity:?}"
|
||||
);
|
||||
assert!(!candidate.buckets_usage.contains_key("bucket"));
|
||||
assert!(!candidate.bucket_sizes.contains_key("bucket"));
|
||||
assert_eq!(candidate.usage_snapshot_converged, Some(false));
|
||||
assert!(candidate.usage_snapshot_admin_projection);
|
||||
assert!(!candidate.is_complete_bucket_usage_snapshot());
|
||||
assert!(!candidate.is_valid_partial_snapshot());
|
||||
}
|
||||
for current in ["", "00000000-0000-0000-0000-000000000000"] {
|
||||
let mut candidate = snapshot();
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.bucket_incarnation = Some(current.to_string());
|
||||
assert_eq!(
|
||||
candidate.retain_current_bucket_usage_for_admin(&HashMap::from([("bucket".to_string(), current.to_string())])),
|
||||
1
|
||||
);
|
||||
assert!(candidate.usage_snapshot_admin_projection);
|
||||
}
|
||||
let mut candidate = snapshot();
|
||||
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&HashMap::new()), 1);
|
||||
assert!(candidate.usage_snapshot_bucket_observations.is_empty());
|
||||
assert_eq!(candidate.usage_snapshot_converged, Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_identity_filter_keeps_healthy_buckets_and_recalculates_partial_totals() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.insert("cold".to_string(), usage(7));
|
||||
candidate.buckets_count = 2;
|
||||
candidate.bucket_sizes.insert("cold".to_string(), 7);
|
||||
candidate.calculate_totals();
|
||||
candidate.replication_info.insert("target".to_string(), Default::default());
|
||||
candidate.tier_stats = Some(Default::default());
|
||||
candidate.unknown_tier_stats = Some(Default::default());
|
||||
assert_eq!(
|
||||
candidate.retain_current_bucket_usage_for_admin(&HashMap::from([("cold".to_string(), INCARNATION.to_string())])),
|
||||
1
|
||||
);
|
||||
assert_eq!(candidate.buckets_usage.len(), 1);
|
||||
assert_eq!(candidate.buckets_usage["cold"].size, 7);
|
||||
assert_eq!(candidate.bucket_sizes, HashMap::from([("cold".to_string(), 7)]));
|
||||
assert_eq!(
|
||||
(
|
||||
candidate.buckets_count,
|
||||
candidate.objects_total_size,
|
||||
candidate.objects_total_count,
|
||||
candidate.versions_total_count
|
||||
),
|
||||
(1, 7, 7, 7)
|
||||
);
|
||||
assert!(candidate.replication_info.is_empty());
|
||||
assert!(candidate.tier_stats.is_none());
|
||||
assert!(candidate.unknown_tier_stats.is_none());
|
||||
assert!(candidate.usage_snapshot_admin_projection);
|
||||
assert_eq!(candidate.usage_snapshot_converged, Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn current_identity_filter_overflow_leaves_totals_unknown() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.insert("large".to_string(), usage(u64::MAX));
|
||||
candidate.buckets_usage.insert("small".to_string(), usage(1));
|
||||
candidate.buckets_count = 3;
|
||||
let current = HashMap::from([
|
||||
("large".to_string(), INCARNATION.to_string()),
|
||||
("small".to_string(), INCARNATION.to_string()),
|
||||
]);
|
||||
assert_eq!(candidate.retain_current_bucket_usage_for_admin(¤t), 3);
|
||||
assert!(candidate.buckets_usage.is_empty());
|
||||
assert!(candidate.bucket_sizes.is_empty());
|
||||
assert_eq!(candidate.usage_snapshot_converged, Some(false));
|
||||
assert!(candidate.usage_snapshot_admin_projection);
|
||||
assert!(!candidate.is_complete_bucket_usage_snapshot());
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Side A is the binary under test on `RUSTFS_S3_STACK=legacy`; side B runs `RUSTFS_E2E_STACK_B`. Set `RUSTFS_E2E_REFERENCE_BINARY` to a released `rustfs` binary to make side A that binary instead (started without `RUSTFS_S3_STACK`, which older releases do not know). A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Both sides run the binary under test on its only (gateway) stack unless `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary, which then runs side A. A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. `RUSTFS_E2E_STACK_B_REGION` starts side B in another region to inject a difference. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
|
||||
The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
|
||||
|
||||
|
||||
@@ -12,14 +12,15 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//! Differential: two `rustfs` processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Stack A runs `RUSTFS_S3_STACK=legacy` of the binary under test, unless `RUSTFS_E2E_REFERENCE_BINARY`
|
||||
//! names a released binary: then A is that binary, started without `RUSTFS_S3_STACK` (older
|
||||
//! releases do not know it), and a bad path fails the run instead of falling back (see
|
||||
//! `reference`). The report names which side is the reference and the `--version` each side ran. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! Side B is the binary under test. Side A is the binary under test too, unless
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released binary: then A is that binary, and a bad path
|
||||
//! fails the run instead of falling back (see `reference`). The report names which side is the
|
||||
//! reference and the `--version` each side ran. `RUSTFS_E2E_STACK_B_REGION`, when set, starts B
|
||||
//! in another region so that a difference can be injected by hand. Neither side is given the
|
||||
//! retired stack selector: the binary under test refuses to start with it, and the gateway stack
|
||||
//! is its only S3 stack. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
//! reports each `(op, field, a, b)` on which the answers differ, and every run writes its report
|
||||
//! to `<target>/dual-stack/<run>.json`. Two processes, not two embedded servers: bucket metadata
|
||||
@@ -37,20 +38,18 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const STACK_A: &str = "legacy";
|
||||
/// Label of a side that runs the binary under test.
|
||||
const UNDER_TEST: &str = "under-test";
|
||||
/// Label of side A in the report when it is the released reference binary.
|
||||
const STACK_A_REFERENCE: &str = "reference";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
const DEFAULT_STACK_B: &str = "legacy";
|
||||
/// The region the negative control gives stack A explicitly, so an inherited `RUSTFS_REGION`
|
||||
/// cannot erase the injected difference. Not `us-east-1`: that region answers GetBucketLocation
|
||||
/// with an empty LocationConstraint and re-creating an owned bucket with 200, so it would not
|
||||
/// report its own name and would add CreateBucket differences to the control.
|
||||
const CONTROL_REGION_A: &str = "us-west-2";
|
||||
const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
/// `normal` cases the legacy stack must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// `normal` cases side A must answer with a 2xx. The comparison alone cannot tell a
|
||||
/// script that exercises real behavior from one in which both stacks fail the same way. Five of
|
||||
/// the 73 fail on legacy by design of this script: PutBucketReplication (no registered remote
|
||||
/// target), GetBucketReplication (so no configuration), PutBucketNotificationConfiguration (no
|
||||
@@ -60,7 +59,6 @@ const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
side_a: SideA,
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
@@ -69,7 +67,6 @@ impl PairConfig {
|
||||
fn from_env() -> Result<Self, BoxError> {
|
||||
Ok(Self {
|
||||
side_a: SideA::from_env()?,
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
})
|
||||
@@ -77,16 +74,16 @@ impl PairConfig {
|
||||
|
||||
fn stack_a_label(&self) -> &'static str {
|
||||
match self.side_a {
|
||||
SideA::SameBinary => STACK_A,
|
||||
SideA::SameBinary => UNDER_TEST,
|
||||
SideA::Reference(_) => STACK_A_REFERENCE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server of the binary under test on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
/// Starts one server of the binary under test, overriding its region only when asked.
|
||||
async fn start_under_test(region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
let mut vars = Vec::new();
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
@@ -94,8 +91,7 @@ async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvi
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts one server of a released binary. `RUSTFS_S3_STACK` is not passed: older releases do not
|
||||
/// know it and have exactly one stack.
|
||||
/// Starts one server of a released binary.
|
||||
async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = Vec::new();
|
||||
@@ -111,11 +107,11 @@ async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTe
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
let side_a = async {
|
||||
match &config.side_a {
|
||||
SideA::SameBinary => start_stack(STACK_A, config.region_a.as_deref()).await,
|
||||
SideA::SameBinary => start_under_test(config.region_a.as_deref()).await,
|
||||
SideA::Reference(binary) => start_reference(binary, config.region_a.as_deref()).await,
|
||||
}
|
||||
};
|
||||
tokio::try_join!(side_a, start_stack(&config.stack_b, config.region_b.as_deref()))
|
||||
tokio::try_join!(side_a, start_under_test(config.region_b.as_deref()))
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
@@ -160,7 +156,7 @@ async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
schema: 2,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: config.stack_a_label().to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
stack_b: UNDER_TEST.to_owned(),
|
||||
reference_side: matches!(config.side_a, SideA::Reference(_)).then_some("a"),
|
||||
version_a,
|
||||
version_b,
|
||||
@@ -235,7 +231,6 @@ mod tests {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
side_a: SideA::SameBinary,
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
};
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Which binary side A of the differential runs, and what it reports about itself.
|
||||
//!
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary to use as the reference. Unset,
|
||||
//! side A is the binary under test on `RUSTFS_S3_STACK=legacy`. Set, it must resolve to an
|
||||
//! side A is the binary under test. Set, it must resolve to an
|
||||
//! executable file or the run fails: a reference that was asked for and silently replaced by the
|
||||
//! binary under test would report an agreement with itself as agreement with the release.
|
||||
|
||||
@@ -28,9 +28,9 @@ pub(super) const ENV_REFERENCE_BINARY: &str = "RUSTFS_E2E_REFERENCE_BINARY";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) enum SideA {
|
||||
/// Both sides are the binary under test; A runs `RUSTFS_S3_STACK=legacy`.
|
||||
/// Both sides are the binary under test.
|
||||
SameBinary,
|
||||
/// A is this released binary, started without `RUSTFS_S3_STACK`.
|
||||
/// A is this released binary.
|
||||
Reference(PathBuf),
|
||||
}
|
||||
|
||||
|
||||
@@ -31,8 +31,8 @@ pub mod chaos;
|
||||
#[cfg(test)]
|
||||
pub mod fake_s3_target;
|
||||
|
||||
// Dual-stack differential (rustfs/backlog#2740): one binary, two processes on
|
||||
// different RUSTFS_S3_STACK values, the same SDK script for every S3 operation,
|
||||
// Differential (rustfs/backlog#2740): two processes, the binary under test against
|
||||
// itself or a released reference binary, the same SDK script for every S3 operation,
|
||||
// compared on status, error code, allowlisted headers and typed body fields.
|
||||
#[cfg(test)]
|
||||
mod dual_stack_differential;
|
||||
@@ -102,6 +102,9 @@ mod replication_target_matrix_test;
|
||||
#[cfg(test)]
|
||||
mod data_usage_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_wide_directory_test;
|
||||
|
||||
// KMS-specific test modules
|
||||
#[cfg(test)]
|
||||
mod kms;
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use aws_sdk_s3::{Client, primitives::ByteStream};
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
use rustfs_data_usage::{BucketUsageValueState, DataUsageInfo};
|
||||
use serde::{Deserialize, de::DeserializeOwned};
|
||||
use tokio::time::{Instant, sleep, timeout};
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, signed_request};
|
||||
use crate::storage_api::scanner_peer_bucket::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
|
||||
type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const BUCKET: &str = "wide-directory-usage";
|
||||
const WIDTH: usize = 512;
|
||||
// The per-disk token bucket allows an initial burst of one second. Even the
|
||||
// first pass can read at most (1 + 3) * 96 = 384 of these 512 root entries.
|
||||
const SCAN_ENV: &[(&str, &str)] = &[
|
||||
("RUST_LOG", "warn,rustfs::scanner::io=debug,rustfs::scanner::data_usage=debug"),
|
||||
("RUSTFS_SCANNER_ENABLED", "true"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS", "3"),
|
||||
("RUSTFS_SCANNER_SPEED", "fastest"),
|
||||
("RUSTFS_SCANNER_DISK_IO_BUDGET_IOPS", "96"),
|
||||
("RUSTFS_SCANNER_DISK_IO_BUDGET_BYTES_PER_SEC", "off"),
|
||||
];
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Status {
|
||||
metrics: Metrics,
|
||||
disk_io_budget: Vec<DiskBudget>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Metrics {
|
||||
current_cycle_active: bool,
|
||||
active_paths: Vec<String>,
|
||||
cycle_max_duration_seconds: f64,
|
||||
partial_cycles_runtime: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DiskBudget {
|
||||
iops: Option<u64>,
|
||||
}
|
||||
|
||||
async fn admin<T: DeserializeOwned>(env: &RustFSTestEnvironment, resource: &str) -> TestResult<T> {
|
||||
let url = format!("{}/rustfs/admin/v3/{resource}", env.url);
|
||||
let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
|
||||
Ok(response.error_for_status()?.json().await?)
|
||||
}
|
||||
|
||||
async fn inventory(client: &Client) -> TestResult<BTreeMap<String, u64>> {
|
||||
let mut pages = client.list_objects_v2().bucket(BUCKET).max_keys(37).into_paginator().send();
|
||||
let mut sizes = BTreeMap::new();
|
||||
while let Some(page) = pages.next().await {
|
||||
for object in page?.contents() {
|
||||
let key = object.key().ok_or("S3 inventory omitted object key")?;
|
||||
let size = u64::try_from(object.size().ok_or("S3 inventory omitted object size")?)?;
|
||||
assert!(sizes.insert(key.to_owned(), size).is_none(), "duplicate inventory key {key}");
|
||||
}
|
||||
}
|
||||
Ok(sizes)
|
||||
}
|
||||
|
||||
async fn scan_status(env: &RustFSTestEnvironment) -> TestResult<Status> {
|
||||
let status: Status = admin(env, "scanner/status").await?;
|
||||
assert_eq!(status.metrics.cycle_max_duration_seconds, 3.0, "effective cycle budget");
|
||||
assert!(status.disk_io_budget.iter().all(|disk| disk.iops == Some(96)), "effective disk budget");
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn certifies_wide_directory_after_mid_scan_process_restart() -> TestResult {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let temporary_log = tempfile::NamedTempFile::new()?;
|
||||
let server_log = env
|
||||
.capture_log_path
|
||||
.get_or_insert_with(|| temporary_log.path().display().to_string())
|
||||
.clone();
|
||||
let bucket_metadata = Path::new(&env.temp_dir)
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(BUCKET_META_PREFIX)
|
||||
.join(BUCKET);
|
||||
// Seed a cold bucket without allowing any earlier, larger-budget sweep to count it.
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[("RUSTFS_SCANNER_ENABLED", "false")])
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(BUCKET).send().await?;
|
||||
stream::iter(0..WIDTH)
|
||||
.map(Ok)
|
||||
.try_for_each_concurrent(12, |i| {
|
||||
let client = &client;
|
||||
async move {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(format!("object-{i:064x}"))
|
||||
.body(ByteStream::from(vec![42; 1 + i % 67]))
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
let expected = inventory(&client).await?;
|
||||
assert_eq!(expected.len(), WIDTH);
|
||||
|
||||
env.restart_server_preserving_data(vec![], SCAN_ENV).await?;
|
||||
let first_pid = env.process.as_ref().ok_or("missing scanner process")?.id();
|
||||
timeout(Duration::from_secs(60), async {
|
||||
loop {
|
||||
let status = scan_status(&env).await?;
|
||||
if status.metrics.partial_cycles_runtime >= 1
|
||||
&& bucket_metadata.join(".usage-cache.bin").join("xl.meta").is_file()
|
||||
&& bucket_metadata.join(".usage-cache.bin.pages").is_dir()
|
||||
&& status.metrics.current_cycle_active
|
||||
&& status
|
||||
.metrics
|
||||
.active_paths
|
||||
.iter()
|
||||
.any(|path| path.split('/').any(|part| part == BUCKET))
|
||||
{
|
||||
let log = tokio::fs::read_to_string(&server_log).await?;
|
||||
if !log
|
||||
.lines()
|
||||
.any(|line| line.contains("partial_saved_not_published") && line.contains(BUCKET))
|
||||
{
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
continue;
|
||||
}
|
||||
assert!(!status.disk_io_budget.is_empty(), "scanner never admitted disk I/O");
|
||||
break Ok::<_, Box<dyn std::error::Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| "scanner did not re-enter the bucket after exhausting the fixed runtime budget")??;
|
||||
let partial: DataUsageInfo = admin(&env, "datausageinfo").await?;
|
||||
assert!(
|
||||
partial
|
||||
.buckets_usage
|
||||
.get(BUCKET)
|
||||
.is_none_or(|value| value.usage_certification.is_none()),
|
||||
"the first process certified the bucket before the injected crash"
|
||||
);
|
||||
|
||||
env.stop_server();
|
||||
// The terminal save has returned and the old process can no longer write.
|
||||
assert!(
|
||||
bucket_metadata.join(".usage-cache.bin").join("xl.meta").is_file()
|
||||
&& bucket_metadata.join(".usage-cache.bin.pages").is_dir(),
|
||||
"the interrupted process must have committed a paged checkpoint"
|
||||
);
|
||||
let restart_log_offset = usize::try_from(tokio::fs::metadata(&server_log).await?.len())?;
|
||||
let restarted_at = SystemTime::now();
|
||||
env.start_rustfs_server_without_cleanup_with_env(SCAN_ENV).await?;
|
||||
let second_pid = env.process.as_ref().ok_or("missing restarted scanner process")?.id();
|
||||
assert_ne!(first_pid, second_pid);
|
||||
let deadline = Instant::now() + Duration::from_secs(180);
|
||||
loop {
|
||||
let usage: DataUsageInfo = admin(&env, "datausageinfo").await?;
|
||||
let status = scan_status(&env).await?;
|
||||
if let Some(bucket) = usage.buckets_usage.get(BUCKET)
|
||||
&& bucket.usage_value_state == Some(BucketUsageValueState::Certified)
|
||||
{
|
||||
let log = tokio::fs::read_to_string(&server_log).await?;
|
||||
let first_admission = log
|
||||
.get(restart_log_offset..)
|
||||
.ok_or("server log truncated across restart")?
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.find(|record| record["state"] == "bucket_checkpoint_prepared" && record["bucket"] == BUCKET)
|
||||
.ok_or("no checkpoint admission in the restarted process")?;
|
||||
assert_eq!(first_admission["resume_position"], true, "restarted scanner discarded the checkpoint");
|
||||
assert!(usage.is_complete_bucket_usage_snapshot());
|
||||
let certificate = bucket
|
||||
.usage_certification
|
||||
.ok_or("missing certificate for the measured bucket")?;
|
||||
assert!(certificate.certified_at >= restarted_at, "stale pre-restart certificate");
|
||||
assert!(status.metrics.partial_cycles_runtime > 0, "new process must cross a cycle deadline");
|
||||
assert!(!status.disk_io_budget.is_empty(), "restarted scanner never admitted disk I/O");
|
||||
assert_eq!(inventory(&client).await?, expected, "durable S3 inventory changed across restart");
|
||||
let count = u64::try_from(expected.len())?;
|
||||
let bytes = expected.values().sum::<u64>();
|
||||
assert_eq!((bucket.objects_count, bucket.size), (count, bytes));
|
||||
assert_eq!((usage.objects_total_count, usage.objects_total_size), (count, bytes));
|
||||
eprintln!(
|
||||
"PIDs {first_pid}/{second_pid}, runtime-limited cycles since restart {}, certificate epoch {} cycle {}, S3 and admin {} objects/{} bytes",
|
||||
status.metrics.partial_cycles_runtime, certificate.epoch, certificate.cycle, count, bytes
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"no certified bucket after restart: {status:?}, value={:?}",
|
||||
usage.buckets_usage.get(BUCKET)
|
||||
);
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
@@ -454,8 +454,8 @@ pub mod data_usage {
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
|
||||
load_admin_data_usage_with_memory_overlay, load_bucket_usage_from_backend_cached, load_compression_total_from_memory,
|
||||
load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
|
||||
load_bucket_usage_from_backend_cached, load_compression_total_from_memory, load_data_usage_from_backend,
|
||||
load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
|
||||
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
|
||||
record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
|
||||
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
|
||||
|
||||
@@ -1405,7 +1405,7 @@ impl BucketMetadata {
|
||||
/// | public access block | Fails closed in `get_public_access_block_config`: degrading grants the anonymous access the operator asked to block. |
|
||||
/// | quota | Fails closed in `get_quota_config`; the enforcement path in `quota::checker` already re-parses the raw JSON and refuses on error. |
|
||||
/// | lifecycle | `get_lifecycle_config` fails closed, so GetBucketLifecycle reports the fault instead of NoSuchLifecycleConfiguration. ILM, scanner and expiry-header consumers still degrade to "no rules": nothing is deleted or moved on the strength of an unreadable rule set, and the bucket keeps serving reads and writes. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup leaves that one bucket's rules unchanged instead of clearing them; other buckets are unaffected. |
|
||||
/// | notification | `get_notification_config` fails closed, so GetBucketNotificationConfiguration reports the fault and startup skips that bucket's rule registration; metadata-bound server dispatch suppresses external delivery for the unreadable snapshot. Other buckets are unaffected. |
|
||||
/// | tagging, CORS, logging, website, accelerate, request payment | The getter fails closed so the matching GET API reports the fault instead of "not configured". Per-request consumers (CORS response headers) still degrade to "not configured", which is the restrictive direction. |
|
||||
/// | bucket ACL | Safe to degrade: it only shapes an optional response. |
|
||||
///
|
||||
|
||||
@@ -2612,6 +2612,10 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cached_metadata(&self) -> Vec<Arc<BucketMetadata>> {
|
||||
self.metadata_map.read().await.values().cloned().collect()
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
self.revoke_pending_bucket_creation(&bucket).await;
|
||||
bump_local_meta_mutation_gen();
|
||||
@@ -5760,6 +5764,64 @@ mod tests {
|
||||
assert_eq!(tags.tag_set.len(), WRITERS, "every concurrent rewrite must survive: {tags:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshots_follow_periodic_refresh_after_lost_peer_signals() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let writer = BucketMetadataSys::new(store.clone());
|
||||
let peer = Arc::new(RwLock::new(BucketMetadataSys::new(store.clone())));
|
||||
let bucket = "notification-periodic-snapshots";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("create empty fixture bucket");
|
||||
}
|
||||
let mut original = BucketMetadata::new(bucket);
|
||||
original.notification_config_xml = concat!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>",
|
||||
"<Queue>arn:rustfs:sqs:us-east-1:inert:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.as_bytes()
|
||||
.to_vec();
|
||||
original.parse_all_configs().expect("parse inert subscription");
|
||||
writer
|
||||
.persist_new_and_set(original)
|
||||
.await
|
||||
.expect("persist initial subscription");
|
||||
peer.read().await.reload_from_store(bucket).await.expect("prime peer cache");
|
||||
let old = peer.read().await.get(bucket).await.expect("cached original snapshot");
|
||||
|
||||
let mut removed = (*old).clone();
|
||||
removed.notification_config_xml = b"<NotificationConfiguration/>".to_vec();
|
||||
removed.parse_all_configs().expect("parse removed subscription");
|
||||
writer
|
||||
.persist_new_and_set(removed)
|
||||
.await
|
||||
.expect("persist removal without peer callback");
|
||||
assert!(Arc::ptr_eq(&old, &peer.read().await.get(bucket).await.expect("peer remains stale")));
|
||||
|
||||
// Exercise the timer's actual one-pass entry point, not the peer RPC.
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
let snapshots = peer.read().await.cached_metadata().await;
|
||||
assert_eq!(snapshots.len(), 1);
|
||||
assert!(!Arc::ptr_eq(&old, &snapshots[0]));
|
||||
assert_eq!(old.bucket_incarnation_id, snapshots[0].bucket_incarnation_id);
|
||||
assert_eq!(snapshots[0].notification_config_xml, b"<NotificationConfiguration/>");
|
||||
assert!(
|
||||
snapshots[0]
|
||||
.notification_config
|
||||
.as_ref()
|
||||
.expect("parsed removal")
|
||||
.behavior()
|
||||
.queue_routes
|
||||
.is_empty()
|
||||
);
|
||||
|
||||
for dir in &dirs {
|
||||
std::fs::remove_dir(dir.path().join(bucket)).expect("remove empty fixture bucket without peer callback");
|
||||
}
|
||||
refresh_buckets_metadata_once(peer.clone()).await;
|
||||
assert!(peer.read().await.cached_metadata().await.is_empty());
|
||||
assert!(peer.read().await.get(bucket).await.is_err());
|
||||
}
|
||||
|
||||
/// Pins the peer reload-notification contract (`reload_from_store`, the
|
||||
/// LoadBucketMetadata RPC path): only metadata actually read from
|
||||
/// persisted storage enters the cache. A load miss errors out and leaves
|
||||
|
||||
@@ -1463,36 +1463,22 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
|
||||
let observed = load_observed_data_usage_snapshot(store.clone()).await.ok().flatten();
|
||||
let (selected, selected_is_current_format) =
|
||||
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
|
||||
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
|
||||
}
|
||||
|
||||
async fn project_admin_data_usage_for_current_buckets(
|
||||
store: Arc<ECStore>,
|
||||
mut response: DataUsageInfo,
|
||||
with_memory_overlay: bool,
|
||||
) -> DataUsageInfo {
|
||||
let mut response = normalize_loaded_data_usage(selected, selected_is_current_format).await.0;
|
||||
use futures::StreamExt as _;
|
||||
let snapshot_measured = response.is_complete_bucket_usage_snapshot() || response.is_valid_partial_snapshot();
|
||||
let overlay_authoritative = with_memory_overlay && snapshot_measured && !runtime_sources::setup_is_dist_erasure().await;
|
||||
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
|
||||
let mut incarnations = HashMap::new();
|
||||
{
|
||||
// Identity reads happen after releasing snapshot and refresh locks,
|
||||
// including on cache hits. Metadata transaction reads linearize each
|
||||
// bucket against creation/deletion without relying on peer notices.
|
||||
// The common deadline cancels unfinished work and bounds added latency.
|
||||
let mut buckets = response
|
||||
.buckets_usage
|
||||
// Display-only identity reads must not serialize every bucket behind a
|
||||
// slow metadata lock while holding the shared admin refresh permit.
|
||||
// The common deadline bounds their total added latency; dropping this
|
||||
// stream cancels all unfinished reads without spawning detached work.
|
||||
// Own the names and store across awaits so this future remains Send
|
||||
// when composed into spawned metrics collectors.
|
||||
let buckets = response
|
||||
.usage_snapshot_bucket_observations
|
||||
.keys()
|
||||
.chain(response.usage_snapshot_bucket_observations.keys())
|
||||
.cloned()
|
||||
.collect::<HashSet<_>>();
|
||||
if overlay_authoritative {
|
||||
// Copy only names while holding the memory lock, then release it
|
||||
// before storage I/O. A later new entry has no identity proof and
|
||||
// will remain unknown in this response.
|
||||
buckets.extend(memory_cache().read().await.keys().cloned());
|
||||
}
|
||||
.collect::<Vec<_>>();
|
||||
let mut identities = futures::stream::iter(buckets)
|
||||
.map(move |bucket| {
|
||||
let store = store.clone();
|
||||
@@ -1506,6 +1492,7 @@ async fn project_admin_data_usage_for_current_buckets(
|
||||
let Ok(Some((bucket, identity))) = tokio::time::timeout_at(deadline, identities.next()).await else {
|
||||
break;
|
||||
};
|
||||
// Unavailable, timed-out or recreated buckets retain canonical values.
|
||||
if let Ok(incarnation) = identity
|
||||
&& !incarnation.is_nil()
|
||||
{
|
||||
@@ -1514,17 +1501,7 @@ async fn project_admin_data_usage_for_current_buckets(
|
||||
}
|
||||
}
|
||||
response.project_bucket_observations_for_admin(&incarnations);
|
||||
if apply_bucket_usage_memory_overlay_if_authoritative(&mut response, overlay_authoritative).await {
|
||||
response.usage_snapshot_complete = false;
|
||||
response.usage_snapshot_partial = true;
|
||||
response.usage_snapshot_converged = Some(false);
|
||||
response.usage_snapshot_admin_projection = true;
|
||||
response.replication_info.clear();
|
||||
response.tier_stats = None;
|
||||
response.unknown_tier_stats = None;
|
||||
}
|
||||
response.retain_current_bucket_usage_for_admin(&incarnations);
|
||||
response
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
|
||||
@@ -1713,7 +1690,11 @@ async fn project_data_usage_from_backend_cached<T>(
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_admin_data_usage_snapshot_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
/// Load the freshest structurally complete snapshot for authenticated admin
|
||||
/// observability. A scan raced by namespace activity may be selected here, but
|
||||
/// never by [`load_data_usage_from_backend_cached`], which remains the
|
||||
/// converged source for quota admission.
|
||||
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
|
||||
|
||||
loop {
|
||||
@@ -1762,38 +1743,6 @@ async fn load_admin_data_usage_snapshot_cached(store: Arc<ECStore>) -> Result<Da
|
||||
}
|
||||
}
|
||||
|
||||
/// Load cached scanner statistics for authenticated admin observability.
|
||||
///
|
||||
/// Each response verifies bucket identities against current storage metadata,
|
||||
/// including cache hits. Missing, failed, or timed-out identity reads leave that
|
||||
/// bucket's usage absent (unknown). Legacy values without an incarnation stay
|
||||
/// unknown until a scanner snapshot certifies their identity. Per-bucket reads
|
||||
/// linearize independently; this is not an atomic namespace snapshot.
|
||||
/// Identity reads have a shared deadline and do not outlive cancellation. The
|
||||
/// unchanged cached baseline remains independent of quota and persistence.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns the cached snapshot loading error when no baseline can be loaded.
|
||||
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
let response = load_admin_data_usage_snapshot_cached(store.clone()).await?;
|
||||
Ok(project_admin_data_usage_for_current_buckets(store, response, false).await)
|
||||
}
|
||||
|
||||
/// Load the admin view with the existing authoritative single-process overlay.
|
||||
///
|
||||
/// Only a complete or valid partial scanner snapshot can receive memory values.
|
||||
/// The final identity check excludes statistics from retired or unverified buckets.
|
||||
/// Distributed nodes continue to use scanner statistics without local totals.
|
||||
/// Cancellation, unknown values, and per-bucket read semantics are the same as
|
||||
/// [`load_admin_data_usage_from_backend_cached`]. No overlay is persisted.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns the cached snapshot loading error when no baseline can be loaded.
|
||||
pub async fn load_admin_data_usage_with_memory_overlay(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
let response = load_admin_data_usage_snapshot_cached(store.clone()).await?;
|
||||
Ok(project_admin_data_usage_for_current_buckets(store, response, true).await)
|
||||
}
|
||||
|
||||
/// Invalidate the process-local persisted usage snapshot cache after a durable
|
||||
/// scanner write.
|
||||
pub async fn invalidate_data_usage_snapshot_cache() {
|
||||
@@ -2856,17 +2805,16 @@ async fn replace_bucket_usage_memory_from_info_if_generation(data_usage_info: &D
|
||||
*cache = next_cache;
|
||||
}
|
||||
|
||||
async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mut DataUsageInfo, authoritative: bool) -> bool {
|
||||
async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mut DataUsageInfo, authoritative: bool) {
|
||||
if !authoritative {
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
let cache = memory_cache().read().await;
|
||||
if cache.is_empty() {
|
||||
return false;
|
||||
return;
|
||||
}
|
||||
|
||||
let was_complete = data_usage_info.usage_snapshot_complete;
|
||||
let persisted_update = data_usage_info.last_update;
|
||||
let mut changed = false;
|
||||
let mut added_bucket = false;
|
||||
@@ -2903,7 +2851,6 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
|
||||
data_usage_info.buckets_count = data_usage_info.buckets_usage.len() as u64;
|
||||
data_usage_info.calculate_totals();
|
||||
}
|
||||
changed || was_complete != data_usage_info.usage_snapshot_complete
|
||||
}
|
||||
|
||||
pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageInfo) {
|
||||
@@ -3797,23 +3744,6 @@ mod tests {
|
||||
*cache_updating().write().await = false;
|
||||
}
|
||||
|
||||
async fn bind_admin_usage_test_identities(store: &Arc<ECStore>, snapshot: &mut DataUsageInfo) {
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
for (bucket, usage) in &mut snapshot.buckets_usage {
|
||||
store
|
||||
.make_bucket(bucket, &crate::storage_api_contracts::bucket::MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create a real bucket for current admin identity validation");
|
||||
usage.bucket_incarnation = Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read the storage-owned fixture identity")
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn data_usage_info_for_test(bucket: &str, objects_count: u64, size: u64, last_update: SystemTime) -> DataUsageInfo {
|
||||
let mut info = DataUsageInfo {
|
||||
last_update: Some(last_update),
|
||||
@@ -4349,11 +4279,7 @@ mod tests {
|
||||
use crate::config::com::save_config;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let (mut authoritative, mut observed) = usage_snapshots_with_missing_observed_set();
|
||||
bind_admin_usage_test_identities(&store, &mut authoritative).await;
|
||||
for (bucket, usage) in &mut observed.buckets_usage {
|
||||
usage.bucket_incarnation = authoritative.buckets_usage[bucket].bucket_incarnation.clone();
|
||||
}
|
||||
let (authoritative, observed) = usage_snapshots_with_missing_observed_set();
|
||||
let primary_bytes = serde_json::to_vec(&authoritative).expect("encode complete usage");
|
||||
let observed_bytes = serde_json::to_vec(&observed).expect("encode partial observation");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), primary_bytes.clone())
|
||||
@@ -4402,11 +4328,6 @@ mod tests {
|
||||
reconciled.scanner_epoch = Some(4);
|
||||
reconciled.scanner_cycle = Some(12);
|
||||
reconciled.usage_snapshot_converged = Some(true);
|
||||
reconciled
|
||||
.buckets_usage
|
||||
.get_mut("large-bucket")
|
||||
.expect("reconciled bucket")
|
||||
.bucket_incarnation = authoritative.buckets_usage["large-bucket"].bucket_incarnation.clone();
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
@@ -4928,12 +4849,6 @@ mod tests {
|
||||
backup.scanner_epoch = Some(4);
|
||||
backup.scanner_cycle = Some(10);
|
||||
backup.usage_snapshot_converged = Some(true);
|
||||
bind_admin_usage_test_identities(&store, &mut current).await;
|
||||
backup
|
||||
.buckets_usage
|
||||
.get_mut("existing")
|
||||
.expect("backup bucket")
|
||||
.bucket_incarnation = current.buckets_usage["existing"].bucket_incarnation.clone();
|
||||
for (path, info) in [
|
||||
(DATA_USAGE_OBJ_NAME_PATH.as_str(), ¤t),
|
||||
(DATA_USAGE_OBJ_BACKUP_PATH.as_str(), &backup),
|
||||
@@ -4956,11 +4871,6 @@ mod tests {
|
||||
after_delete.scanner_epoch = current.scanner_epoch;
|
||||
after_delete.scanner_cycle = current.scanner_cycle;
|
||||
after_delete.usage_snapshot_converged = Some(true);
|
||||
after_delete
|
||||
.buckets_usage
|
||||
.get_mut("existing")
|
||||
.expect("post-delete bucket")
|
||||
.bucket_incarnation = current.buckets_usage["existing"].bucket_incarnation.clone();
|
||||
let mut stale_observation =
|
||||
data_usage_info_for_test("retired-incarnation", 99, 999, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
|
||||
stale_observation.scanner_epoch = Some(4);
|
||||
@@ -5022,7 +4932,6 @@ mod tests {
|
||||
authoritative.scanner_epoch = Some(7);
|
||||
authoritative.scanner_cycle = Some(10);
|
||||
authoritative.usage_snapshot_converged = Some(true);
|
||||
bind_admin_usage_test_identities(&store, &mut authoritative).await;
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
@@ -5037,10 +4946,6 @@ mod tests {
|
||||
info.scanner_cycle = Some(cycle);
|
||||
info.usage_snapshot_converged = Some(false);
|
||||
info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
|
||||
info.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("observed bucket")
|
||||
.bucket_incarnation = authoritative.buckets_usage["bucket"].bucket_incarnation.clone();
|
||||
info
|
||||
};
|
||||
let first = observation(7, 11, 200, 11, 1_100);
|
||||
|
||||
@@ -221,7 +221,7 @@ async fn companion_admin_cache_isolated_from_quota_memory_and_persistence() {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_buckets() {
|
||||
async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buckets() {
|
||||
use crate::bucket::metadata_sys::{
|
||||
acquire_metadata_transaction_guard, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
@@ -280,11 +280,8 @@ async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_bucke
|
||||
.await
|
||||
.expect("optional projection must return before the blocked metadata lock's long timeout")
|
||||
.expect("canonical admin response stays available");
|
||||
assert!(
|
||||
!response.buckets_usage.contains_key(blocked),
|
||||
"an unavailable current identity cannot bind the canonical value"
|
||||
);
|
||||
assert!(!response.bucket_sizes.contains_key(blocked));
|
||||
assert_eq!(response.buckets_usage[blocked].size, 100);
|
||||
assert_eq!(response.buckets_usage[blocked].usage_value_state, Some(BucketUsageValueState::Carried));
|
||||
assert_eq!(response.buckets_usage[healthy].size, 250);
|
||||
assert_eq!(response.buckets_usage[healthy].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
@@ -296,12 +293,13 @@ async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_bucke
|
||||
.is_ok()
|
||||
);
|
||||
drop(blocked_guard);
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let recovered = load_admin_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("refresh after lock recovery");
|
||||
assert_eq!(
|
||||
recovered.buckets_usage[blocked].size, 250,
|
||||
"unavailable identity can recover on a cache hit"
|
||||
"unavailable identity can recover on the next refresh"
|
||||
);
|
||||
assert_eq!(recovered.buckets_usage[healthy].size, 250);
|
||||
let quota = load_data_usage_from_backend_cached(store)
|
||||
@@ -310,249 +308,3 @@ async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_bucke
|
||||
assert_eq!(quota.buckets_usage[blocked].size, 100);
|
||||
assert_eq!(quota.buckets_usage[healthy].size, 100);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn cached_canonical_admin_usage_rechecks_current_identity_without_peer_invalidation() {
|
||||
use crate::bucket::metadata::{load_bucket_incarnation, save_bucket_incarnation};
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::config::com::save_config;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "usage-recreated-cache";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create identity fixture");
|
||||
let old_identity = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("original identity");
|
||||
let mut canonical = companion_snapshot(bucket, old_identity);
|
||||
canonical.usage_snapshot_bucket_observations.clear();
|
||||
canonical.usage_snapshot_converged = Some(true);
|
||||
canonical.usage_snapshot_pending_buckets.clear();
|
||||
canonical.usage_snapshot_pending_bucket_count = 0;
|
||||
canonical
|
||||
.buckets_usage
|
||||
.get_mut(bucket)
|
||||
.expect("canonical bucket")
|
||||
.objects_count = 107;
|
||||
canonical.calculate_totals();
|
||||
let original_bytes = serde_json::to_vec(&canonical).expect("encode canonical old bucket");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), original_bytes.clone())
|
||||
.await
|
||||
.expect("persist canonical root");
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let first = load_admin_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("prime raw admin cache");
|
||||
assert_eq!(first.buckets_usage[bucket].objects_count, 107);
|
||||
assert!(first.is_complete_bucket_usage_snapshot());
|
||||
let cache_generation = admin_data_usage_snapshot_generation();
|
||||
let new_identity = uuid::Uuid::new_v4();
|
||||
// Simulate the storage-owned identity changing on another node while this
|
||||
// process has not received its invalidation notice. Do not clear the cache.
|
||||
save_bucket_incarnation(store.clone(), bucket, new_identity)
|
||||
.await
|
||||
.expect("publish changed disk identity");
|
||||
assert_eq!(
|
||||
load_bucket_incarnation(store.clone(), bucket)
|
||||
.await
|
||||
.expect("read new sidecar"),
|
||||
Some(new_identity)
|
||||
);
|
||||
let second = load_admin_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("read raw cache with fresh identity");
|
||||
assert_eq!(admin_data_usage_snapshot_generation(), cache_generation);
|
||||
assert!(
|
||||
!second.buckets_usage.contains_key(bucket),
|
||||
"same name cannot retain the predecessor's 107"
|
||||
);
|
||||
assert!(!second.bucket_sizes.contains_key(bucket));
|
||||
assert_eq!(second.usage_snapshot_converged, Some(false));
|
||||
assert!(second.usage_snapshot_admin_projection);
|
||||
assert!(reject_admin_usage_projection(&second).is_err());
|
||||
assert!(certified_bucket_usage(&second, bucket, new_identity).is_none());
|
||||
assert!(parse_usage_snapshot(&serde_json::to_vec(&second).expect("encode display response")).is_err());
|
||||
let quota = load_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("independent quota baseline");
|
||||
assert_eq!(quota.buckets_usage[bucket].objects_count, 107);
|
||||
assert_eq!(
|
||||
read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("unchanged persisted root"),
|
||||
original_bytes
|
||||
);
|
||||
// Rebinding a cached raw baseline is transient: recovery does not need to
|
||||
// reload or permanently blank a healthy bucket's statistics.
|
||||
save_bucket_incarnation(store.clone(), bucket, old_identity)
|
||||
.await
|
||||
.expect("restore fixture identity");
|
||||
let recovered = load_admin_data_usage_from_backend_cached(store)
|
||||
.await
|
||||
.expect("recover on another cache hit");
|
||||
assert_eq!(recovered.buckets_usage[bucket].objects_count, 107);
|
||||
assert!(recovered.is_complete_bucket_usage_snapshot());
|
||||
assert_eq!(admin_data_usage_snapshot_generation(), cache_generation);
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn admin_memory_overlay_requires_a_measured_scanner_baseline() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::config::com::save_config;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
assert!(!runtime_sources::setup_is_dist_erasure().await);
|
||||
let bucket = "usage-unmeasured-overlay";
|
||||
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap();
|
||||
let mut baseline = data_usage_info_for_test(bucket, 1, 100, SystemTime::UNIX_EPOCH);
|
||||
let usage = baseline.buckets_usage.get_mut(bucket).unwrap();
|
||||
usage.bucket_incarnation = Some(incarnation.to_string());
|
||||
let memory_usage = BucketUsageInfo {
|
||||
size: 150,
|
||||
..usage.clone()
|
||||
};
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
replace_bucket_usage_memory_from_authoritative(bucket, memory_usage, SystemTime::now()).await;
|
||||
|
||||
// A missing snapshot and a mismatched bucket count both leave usage unknown.
|
||||
for (buckets_count, expected_size) in [(None, None), (Some(2), None), (Some(1), Some(150))] {
|
||||
if let Some(count) = buckets_count {
|
||||
baseline.buckets_count = count;
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&baseline).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let response = load_admin_data_usage_with_memory_overlay(store.clone()).await.unwrap();
|
||||
assert_eq!(
|
||||
response.buckets_usage.get(bucket).map(|usage| usage.size),
|
||||
expected_size,
|
||||
"baseline count {buckets_count:?}"
|
||||
);
|
||||
assert_eq!(response.bucket_sizes.get(bucket).copied(), expected_size);
|
||||
assert_eq!(response.objects_total_size, expected_size.unwrap_or_default());
|
||||
assert_eq!(response.usage_snapshot_admin_projection, expected_size.is_some());
|
||||
assert!(!response.is_complete_bucket_usage_snapshot());
|
||||
assert!(!response.is_valid_partial_snapshot());
|
||||
}
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn admin_memory_overlay_cannot_reintroduce_a_predecessor_identity() {
|
||||
use crate::bucket::metadata::save_bucket_incarnation;
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::config::com::save_config;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "usage-recreated-overlay";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create overlay identity fixture");
|
||||
let old_identity = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("original identity");
|
||||
let mut canonical = companion_snapshot(bucket, old_identity);
|
||||
canonical.usage_snapshot_bucket_observations.clear();
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&canonical).expect("encode canonical root"),
|
||||
)
|
||||
.await
|
||||
.expect("persist overlay fixture");
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let cached_usage = canonical.buckets_usage[bucket].clone();
|
||||
replace_bucket_usage_memory_from_authoritative(bucket, cached_usage, SystemTime::now()).await;
|
||||
let new_identity = uuid::Uuid::new_v4();
|
||||
save_bucket_incarnation(store.clone(), bucket, new_identity)
|
||||
.await
|
||||
.expect("publish a replacement identity");
|
||||
let response = load_admin_data_usage_with_memory_overlay(store.clone())
|
||||
.await
|
||||
.expect("project after memory overlay");
|
||||
assert!(!response.buckets_usage.contains_key(bucket));
|
||||
assert_eq!(response.usage_snapshot_converged, Some(false));
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn admin_overlay_flags_only_change_keeps_identity_bound_cold_usage() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::config::com::save_config;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
assert!(
|
||||
!runtime_sources::setup_is_dist_erasure().await,
|
||||
"exercise the production single-process overlay"
|
||||
);
|
||||
let cold = "usage-overlay-cold";
|
||||
let unverified = "usage-overlay-unverified";
|
||||
for bucket in [cold, unverified] {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create overlay fixture bucket");
|
||||
}
|
||||
let identity = store
|
||||
.bucket_incarnation_id_from_disk(cold)
|
||||
.await
|
||||
.expect("cold bucket identity");
|
||||
let mut canonical = companion_snapshot(cold, identity);
|
||||
canonical.usage_snapshot_bucket_observations.clear();
|
||||
canonical.usage_snapshot_converged = Some(true);
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&canonical).expect("encode cold canonical value"),
|
||||
)
|
||||
.await
|
||||
.expect("persist cold baseline");
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
memory_cache().write().await.insert(
|
||||
unverified.to_string(),
|
||||
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
|
||||
);
|
||||
record_bucket_object_write_memory(unverified, None, 42).await;
|
||||
{
|
||||
let cache = memory_cache().read().await;
|
||||
assert!(cache[unverified].dirty);
|
||||
assert!(!cache[unverified].authoritative);
|
||||
}
|
||||
let response = load_admin_data_usage_with_memory_overlay(store)
|
||||
.await
|
||||
.expect("identity-filtered flags-only overlay");
|
||||
assert_eq!(response.buckets_usage[cold].size, 100);
|
||||
assert_eq!(response.objects_total_size, 100);
|
||||
assert_eq!(response.bucket_sizes[cold], 100);
|
||||
assert!(!response.buckets_usage.contains_key(unverified));
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
assert_eq!(response.usage_snapshot_converged, Some(false));
|
||||
assert!(!response.is_complete_bucket_usage_snapshot());
|
||||
assert!(!response.is_valid_partial_snapshot());
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
@@ -1033,26 +1033,16 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn server_info_reads_complete_observation_without_promoting_authority() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
||||
use crate::config::com::{read_config, save_config};
|
||||
use crate::data_usage::{
|
||||
DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, invalidate_data_usage_snapshot_cache,
|
||||
load_data_usage_from_backend_cached,
|
||||
};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket("bucket", &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create a real observed bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk("bucket")
|
||||
.await
|
||||
.expect("read current storage-owned incarnation");
|
||||
let primary = DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
||||
scanner_epoch: Some(1),
|
||||
@@ -1076,7 +1066,6 @@ mod tests {
|
||||
versions_count: 4,
|
||||
delete_markers_count: 1,
|
||||
size: 6,
|
||||
bucket_incarnation: Some(incarnation.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)]
|
||||
@@ -1128,46 +1117,6 @@ mod tests {
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_filtered_usage_remains_unknown_in_server_info() {
|
||||
let mut info = rustfs_data_usage::DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
|
||||
buckets_count: 1,
|
||||
objects_total_count: 107,
|
||||
objects_total_size: 1024,
|
||||
buckets_usage: [(
|
||||
"bucket".to_string(),
|
||||
rustfs_data_usage::BucketUsageInfo {
|
||||
objects_count: 107,
|
||||
size: 1024,
|
||||
bucket_incarnation: Some("11111111-1111-4111-8111-111111111111".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
usage_snapshot_complete: true,
|
||||
..Default::default()
|
||||
};
|
||||
let current =
|
||||
std::collections::HashMap::from([("bucket".to_string(), "22222222-2222-4222-8222-222222222222".to_string())]);
|
||||
assert_eq!(info.retain_current_bucket_usage_for_admin(¤t), 1);
|
||||
assert!(info.buckets_usage.is_empty());
|
||||
assert!(info.usage_snapshot_admin_projection);
|
||||
assert_eq!(info.usage_snapshot_converged, Some(false));
|
||||
let mut buckets = rustfs_madmin::Buckets::default();
|
||||
let mut objects = rustfs_madmin::Objects::default();
|
||||
let mut versions = rustfs_madmin::Versions::default();
|
||||
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
|
||||
let mut usage = rustfs_madmin::Usage::default();
|
||||
apply_data_usage_result(Ok(info), &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage);
|
||||
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
assert_eq!(objects.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
assert_eq!(versions.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
assert_eq!(delete_markers.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_data_usage_is_unavailable_in_server_info() {
|
||||
let mut buckets = rustfs_madmin::Buckets::default();
|
||||
|
||||
@@ -158,6 +158,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_scanner_metrics::metrics::ScannerMe
|
||||
last_cycle_lifecycle_transition_actions: metrics.last_cycle_lifecycle_transition_actions,
|
||||
last_cycle_partial_source: metrics.last_cycle_partial_source,
|
||||
last_cycle_partial_source_code: metrics.last_cycle_partial_source_code,
|
||||
last_cycle_partial_stage: metrics.last_cycle_partial_stage,
|
||||
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
|
||||
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
|
||||
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
|
||||
@@ -685,6 +686,7 @@ mod test {
|
||||
current_started,
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
last_cycle_partial_stage: Some("object_budget".into()),
|
||||
cycle_timeout_total: 3,
|
||||
cycle_recovery_required_total: 2,
|
||||
cycle_last_progress_age: 17,
|
||||
@@ -700,6 +702,7 @@ mod test {
|
||||
assert_eq!(scanner.current_started, current_started);
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
assert_eq!(scanner.last_cycle_partial_stage.as_deref(), Some("object_budget"));
|
||||
assert_eq!(scanner.cycle_timeout_total, 3);
|
||||
assert_eq!(scanner.cycle_recovery_required_total, 2);
|
||||
assert_eq!(scanner.cycle_last_progress_age, 17);
|
||||
|
||||
@@ -775,6 +775,15 @@ impl ECStore {
|
||||
sys.read().await.get(bucket).await
|
||||
}
|
||||
|
||||
/// Snapshots this instance's cached bucket metadata without disk IO.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when this instance's metadata system is not initialized.
|
||||
pub async fn cached_bucket_metadata(&self) -> Result<Vec<Arc<BucketMetadata>>> {
|
||||
let sys = metadata_sys::require_bucket_metadata_sys_in(&self.ctx)?;
|
||||
Ok(sys.read().await.cached_metadata().await)
|
||||
}
|
||||
|
||||
/// Resolve stored tags for authorization. Only confirmed absence returns an
|
||||
/// empty map; unavailable, fabricated or malformed metadata must not grant access.
|
||||
pub async fn get_bucket_tags_for_policy(&self, bucket: &str) -> Result<HashMap<String, String>> {
|
||||
|
||||
@@ -709,6 +709,8 @@ pub struct ScannerMetrics {
|
||||
pub last_cycle_partial_source: String,
|
||||
#[serde(rename = "last_cycle_partial_source_code", default)]
|
||||
pub last_cycle_partial_source_code: u64,
|
||||
#[serde(rename = "last_cycle_partial_stage", default, skip_serializing_if = "Option::is_none")]
|
||||
pub last_cycle_partial_stage: Option<String>,
|
||||
#[serde(rename = "last_cycle_duration_seconds", default)]
|
||||
pub last_cycle_duration_seconds: f64,
|
||||
#[serde(rename = "last_cycle_objects_scanned", default)]
|
||||
@@ -853,6 +855,7 @@ impl ScannerMetrics {
|
||||
self.last_cycle_partial_reason_code = other.last_cycle_partial_reason_code;
|
||||
self.last_cycle_partial_source = other.last_cycle_partial_source.clone();
|
||||
self.last_cycle_partial_source_code = other.last_cycle_partial_source_code;
|
||||
self.last_cycle_partial_stage.clone_from(&other.last_cycle_partial_stage);
|
||||
self.last_cycle_duration_seconds = other.last_cycle_duration_seconds;
|
||||
self.throttle_idle_mode_enabled = other.throttle_idle_mode_enabled;
|
||||
self.throttle_sleep_factor = other.throttle_sleep_factor;
|
||||
@@ -2138,6 +2141,7 @@ mod tests {
|
||||
collected_at: collected_at + SignedDuration::from_secs(1),
|
||||
last_cycle_partial_source: "lifecycle".to_string(),
|
||||
last_cycle_partial_source_code: 2,
|
||||
last_cycle_partial_stage: Some("object_budget".into()),
|
||||
pacing_pressure: ScannerPacingPressureSnapshot {
|
||||
primary_pressure: "queued_scans".to_string(),
|
||||
current_queued_scans: 4,
|
||||
@@ -2162,6 +2166,7 @@ mod tests {
|
||||
|
||||
assert_eq!(scanner.last_cycle_partial_source, "lifecycle");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 2);
|
||||
assert_eq!(scanner.last_cycle_partial_stage.as_deref(), Some("object_budget"));
|
||||
assert_eq!(scanner.pacing_pressure.primary_pressure, "queued_scans");
|
||||
assert_eq!(scanner.pacing_pressure.current_queued_scans, 4);
|
||||
assert_eq!(scanner.pacing_pressure.current_active_scans, 3);
|
||||
|
||||
@@ -47,12 +47,18 @@ impl NotifyBucketConfigManager {
|
||||
}
|
||||
|
||||
pub async fn has_subscriber(&self, bucket: &str, event: &EventName) -> bool {
|
||||
if !self.subscriber_view.has_subscriber(bucket, event) {
|
||||
// The standalone index is only a fast negative check for name-only
|
||||
// rules. Metadata-bound rules can change without an index callback.
|
||||
if !self.rule_engine.uses_bucket_metadata_source() && !self.subscriber_view.has_subscriber(bucket, event) {
|
||||
return false;
|
||||
}
|
||||
self.rule_engine.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
pub(crate) fn bind_bucket_metadata_source(&self, store: &Arc<crate::NotifyStore>) {
|
||||
self.rule_engine.bind_bucket_metadata_source(store);
|
||||
}
|
||||
|
||||
/// Validates that the runtime can accept `cfg` for `bucket` without
|
||||
/// mutating any state.
|
||||
pub async fn validate_bucket_notification_config(
|
||||
|
||||
@@ -249,6 +249,16 @@ impl NotificationSystem {
|
||||
self.services.bucket_config_manager.has_subscriber(bucket, event).await
|
||||
}
|
||||
|
||||
/// Binds server dispatch to the store-owned metadata cache. Local writes,
|
||||
/// peer reloads, deletes and periodic metadata refreshes then select rules
|
||||
/// from the same snapshot; delayed name-only rule replacements are ignored
|
||||
/// and clears only evict compiled snapshots. Missing/unreadable metadata or
|
||||
/// a dropped store suppresses external dispatch without falling back to the
|
||||
/// standalone rules. Already queued or in-flight deliveries are not recalled.
|
||||
pub fn bind_bucket_metadata_source(&self, store: &Arc<crate::NotifyStore>) {
|
||||
self.services.bucket_config_manager.bind_bucket_metadata_source(store);
|
||||
}
|
||||
|
||||
/// Returns true when at least one in-process consumer is subscribed to live events.
|
||||
pub fn has_live_listeners(&self) -> bool {
|
||||
self.services.pipeline.has_live_listeners()
|
||||
|
||||
@@ -12,12 +12,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::rules::xml_config::{FilterRule, S3KeyFilter};
|
||||
use crate::rules::{RulesMap, TargetIdSet};
|
||||
use crate::storage_api::bucket_metadata::{BucketMetadata, NotifyStore};
|
||||
use percent_encoding::percent_decode_str;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use rustfs_targets::arn::{ARN, TargetID};
|
||||
use starshard::{AsyncShardedHashMap, DEFAULT_SHARDS, SnapshotMode};
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, RwLock, Weak};
|
||||
use tracing::info;
|
||||
|
||||
const LOG_COMPONENT_NOTIFY: &str = "notify";
|
||||
@@ -35,20 +37,82 @@ fn decoded_object_key_for_matching(object_key: &str) -> Option<String> {
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct NotifyRuleEngine {
|
||||
bucket_rules_map: Arc<AsyncShardedHashMap<String, RulesMap, rustc_hash::FxBuildHasher>>,
|
||||
bucket_rules_map: Arc<AsyncShardedHashMap<String, CachedBucketRules, rustc_hash::FxBuildHasher>>,
|
||||
// Never hold this short synchronous guard across an await. Metadata cache
|
||||
// reads precede the compiled-rules cache, so there is no reverse lock order.
|
||||
metadata_source: Arc<RwLock<Option<Weak<NotifyStore>>>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CachedBucketRules {
|
||||
metadata: Option<Weak<BucketMetadata>>,
|
||||
// Cache rejected snapshots too, so corrupt persisted routes are not
|
||||
// repeatedly compiled on the event hot path.
|
||||
rules: Option<RulesMap>,
|
||||
}
|
||||
|
||||
fn rules_from_metadata(metadata: &BucketMetadata) -> Option<RulesMap> {
|
||||
let mut rules = RulesMap::new();
|
||||
let Some(config) = metadata.notification_config.as_ref() else {
|
||||
// Unreadable metadata is not authority to keep an old subscription.
|
||||
return metadata.notification_config_xml.is_empty().then_some(rules);
|
||||
};
|
||||
let config = config.behavior();
|
||||
for route in config
|
||||
.queue_routes
|
||||
.into_iter()
|
||||
.chain(config.topic_routes)
|
||||
.chain(config.lambda_routes)
|
||||
{
|
||||
let target = ARN::parse(&route.destination).ok()?.target_id;
|
||||
let events: Vec<_> = route.events.iter().filter_map(|event| EventName::parse(event).ok()).collect();
|
||||
let mut filter = S3KeyFilter::default();
|
||||
for (name, value) in route.filter_rules.into_iter().flatten() {
|
||||
filter.filter_rule_list.push(FilterRule {
|
||||
name: name?,
|
||||
value: value?,
|
||||
});
|
||||
}
|
||||
filter.validate().ok()?;
|
||||
rules.add_rule_config(&events, filter.pattern(), target);
|
||||
}
|
||||
Some(rules)
|
||||
}
|
||||
|
||||
impl NotifyRuleEngine {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
bucket_rules_map: Arc::new(AsyncShardedHashMap::with_snapshot_mode(DEFAULT_SHARDS, SnapshotMode::Cached)),
|
||||
metadata_source: Arc::new(RwLock::new(None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Uses this store's immutable metadata snapshots as rule authority. Once
|
||||
/// bound, missing metadata or a dropped store fails closed, never falling
|
||||
/// back to the standalone name-only table. The weak handle avoids cycles.
|
||||
pub fn bind_bucket_metadata_source(&self, store: &Arc<NotifyStore>) {
|
||||
*self.metadata_source.write().unwrap_or_else(|err| err.into_inner()) = Some(Arc::downgrade(store));
|
||||
}
|
||||
|
||||
fn metadata_source(&self) -> Option<Weak<NotifyStore>> {
|
||||
self.metadata_source.read().unwrap_or_else(|err| err.into_inner()).clone()
|
||||
}
|
||||
|
||||
pub(crate) fn uses_bucket_metadata_source(&self) -> bool {
|
||||
self.metadata_source().is_some()
|
||||
}
|
||||
|
||||
pub async fn is_target_bound_to_any_bucket(&self, target_id: &TargetID) -> bool {
|
||||
if let Some(source) = self.metadata_source() {
|
||||
let Some(store) = source.upgrade() else { return true };
|
||||
let Ok(metadata) = store.cached_bucket_metadata().await else { return true };
|
||||
return metadata
|
||||
.iter()
|
||||
.any(|metadata| rules_from_metadata(metadata).is_none_or(|rules| rules.contains_target_id(target_id)));
|
||||
}
|
||||
let items = self.bucket_rules_map.iter().await;
|
||||
for (_bucket, rules_map) in items {
|
||||
if rules_map.contains_target_id(target_id) {
|
||||
if rules_map.rules.is_some_and(|rules| rules.contains_target_id(target_id)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -56,11 +120,24 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn set_bucket_rules(&self, bucket: &str, rules_map: RulesMap) {
|
||||
if self.uses_bucket_metadata_source() {
|
||||
// Delayed PUT/startup callbacks cannot override a newer revision or
|
||||
// a same-name replacement. Only published metadata is authoritative.
|
||||
return;
|
||||
}
|
||||
let event_count = rules_map.iter_events().count();
|
||||
if rules_map.is_empty() {
|
||||
self.bucket_rules_map.remove(&bucket.to_string()).await;
|
||||
} else {
|
||||
self.bucket_rules_map.insert(bucket.to_string(), rules_map).await;
|
||||
self.bucket_rules_map
|
||||
.insert(
|
||||
bucket.to_string(),
|
||||
CachedBucketRules {
|
||||
metadata: None,
|
||||
rules: Some(rules_map),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
info!(
|
||||
event = EVENT_NOTIFY_RULES_UPDATED,
|
||||
@@ -74,7 +151,47 @@ impl NotifyRuleEngine {
|
||||
}
|
||||
|
||||
pub async fn get_bucket_rules(&self, bucket: &str) -> Option<RulesMap> {
|
||||
self.bucket_rules_map.get(&bucket.to_string()).await
|
||||
if let Some(source) = self.metadata_source() {
|
||||
let metadata = match source.upgrade() {
|
||||
Some(store) => store.get_bucket_metadata(bucket).await.ok(),
|
||||
None => None,
|
||||
};
|
||||
return self.rules_for_metadata(bucket, metadata).await;
|
||||
}
|
||||
self.bucket_rules_map
|
||||
.get(&bucket.to_string())
|
||||
.await
|
||||
.and_then(|cached| cached.rules)
|
||||
}
|
||||
|
||||
async fn rules_for_metadata(&self, bucket: &str, metadata: Option<Arc<BucketMetadata>>) -> Option<RulesMap> {
|
||||
let key = bucket.to_owned();
|
||||
let Some(metadata) = metadata else {
|
||||
self.bucket_rules_map.remove(&key).await;
|
||||
return None;
|
||||
};
|
||||
if let Some(cached) = self.bucket_rules_map.get(&key).await
|
||||
&& cached
|
||||
.metadata
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| Weak::ptr_eq(snapshot, &Arc::downgrade(&metadata)))
|
||||
{
|
||||
return cached.rules;
|
||||
}
|
||||
// Snapshot identity covers incarnation, revision AND bytes, without
|
||||
// trusting wall-clock ordering. A late compilation may evict a newer
|
||||
// cache entry, but cannot be reused by a reader of a different snapshot.
|
||||
let rules = rules_from_metadata(&metadata);
|
||||
self.bucket_rules_map
|
||||
.insert(
|
||||
key,
|
||||
CachedBucketRules {
|
||||
metadata: Some(Arc::downgrade(&metadata)),
|
||||
rules: rules.clone(),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
rules
|
||||
}
|
||||
|
||||
pub async fn clear_bucket_rules(&self, bucket: &str) {
|
||||
@@ -119,8 +236,177 @@ impl Default for NotifyRuleEngine {
|
||||
mod tests {
|
||||
use super::NotifyRuleEngine;
|
||||
use crate::rules::RulesMap;
|
||||
use crate::storage_api::bucket_metadata::{BucketMetadata, deserialize_bucket_config};
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
fn metadata(target: &str) -> Arc<BucketMetadata> {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.notification_config_xml = format!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>\
|
||||
<Queue>arn:rustfs:sqs:us-east-1:{target}:webhook</Queue></QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.into_bytes();
|
||||
metadata.notification_config =
|
||||
Some(deserialize_bucket_config(&metadata.notification_config_xml).expect("parse inert notification fixture"));
|
||||
Arc::new(metadata)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshot_cache_never_reuses_replaced_incarnation_or_revision() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let old = metadata("old");
|
||||
let mut replacement = (*metadata("new")).clone();
|
||||
// Identical timestamps must not make different configurations equal.
|
||||
replacement.notification_config_updated_at = old.notification_config_updated_at;
|
||||
let replacement = Arc::new(replacement);
|
||||
assert_ne!(old.bucket_incarnation_id, replacement.bucket_incarnation_id);
|
||||
let old_target = TargetID::new("old".into(), "webhook".into());
|
||||
let new_target = TargetID::new("new".into(), "webhook".into());
|
||||
|
||||
for (snapshot, wanted, unwanted) in [
|
||||
(old.clone(), &old_target, &new_target),
|
||||
(replacement.clone(), &new_target, &old_target),
|
||||
// Simulate a late compilation of a captured, older snapshot.
|
||||
(old.clone(), &old_target, &new_target),
|
||||
(replacement.clone(), &new_target, &old_target),
|
||||
] {
|
||||
let rules = engine
|
||||
.rules_for_metadata("bucket", Some(snapshot))
|
||||
.await
|
||||
.expect("readable snapshot");
|
||||
let targets = rules.match_rules(EventName::ObjectCreatedPut, "object");
|
||||
assert!(targets.contains(wanted));
|
||||
assert!(!targets.contains(unwanted));
|
||||
}
|
||||
|
||||
let mut next_revision = (*old).clone();
|
||||
next_revision.notification_config = replacement.notification_config.clone();
|
||||
next_revision.notification_config_xml = replacement.notification_config_xml.clone();
|
||||
let rules = engine
|
||||
.rules_for_metadata("bucket", Some(Arc::new(next_revision)))
|
||||
.await
|
||||
.expect("same-incarnation revision");
|
||||
assert!(!rules.contains_target_id(&old_target));
|
||||
assert!(rules.contains_target_id(&new_target));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshot_absence_and_corruption_never_fall_back_to_old_rules() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let old = metadata("old");
|
||||
assert!(engine.rules_for_metadata("bucket", Some(old.clone())).await.is_some());
|
||||
|
||||
let mut unreadable = (*old).clone();
|
||||
unreadable.notification_config = None;
|
||||
unreadable.notification_config_xml = b"<broken>".to_vec();
|
||||
assert!(
|
||||
engine
|
||||
.rules_for_metadata("bucket", Some(Arc::new(unreadable)))
|
||||
.await
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
engine
|
||||
.bucket_rules_map
|
||||
.get(&"bucket".to_owned())
|
||||
.await
|
||||
.expect("cache rejected snapshot")
|
||||
.rules
|
||||
.is_none()
|
||||
);
|
||||
assert!(engine.rules_for_metadata("bucket", None).await.is_none());
|
||||
assert!(engine.bucket_rules_map.get(&"bucket".to_owned()).await.is_none());
|
||||
|
||||
// A recreated bucket without notifications has no inherited targets.
|
||||
let replacement = Arc::new(BucketMetadata::new("bucket"));
|
||||
assert!(
|
||||
engine
|
||||
.rules_for_metadata("bucket", Some(replacement))
|
||||
.await
|
||||
.expect("empty snapshot")
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn notification_snapshot_binding_never_falls_back_after_store_loss() {
|
||||
let engine = NotifyRuleEngine::new();
|
||||
let target = TargetID::new("old".into(), "webhook".into());
|
||||
let mut rules = RulesMap::new();
|
||||
rules.add_rule_config(&[EventName::ObjectCreatedPut], "*".into(), target.clone());
|
||||
engine.set_bucket_rules("bucket", rules.clone()).await;
|
||||
assert!(engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
|
||||
*engine.metadata_source.write().expect("metadata source lock") = Some(Weak::new());
|
||||
// A delayed validated PUT must not revive the standalone table, even
|
||||
// when its owning store is gone. Target removal also fails closed.
|
||||
engine.set_bucket_rules("bucket", rules).await;
|
||||
assert!(!engine.has_subscriber("bucket", &EventName::ObjectCreatedPut).await);
|
||||
assert!(
|
||||
engine
|
||||
.match_targets("bucket", EventName::ObjectCreatedPut, "object")
|
||||
.await
|
||||
.is_empty()
|
||||
);
|
||||
assert!(engine.is_target_bound_to_any_bucket(&target).await);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_snapshot_compilation_preserves_destinations_events_and_filters() {
|
||||
let mut metadata = (*metadata("queue")).clone();
|
||||
metadata.notification_config_xml = concat!(
|
||||
"<NotificationConfiguration>",
|
||||
"<QueueConfiguration><Event>s3:ObjectCreated:*</Event><Event>s3:Future:Unrecognized</Event>",
|
||||
"<Queue>arn:rustfs:sqs:us-east-1:queue:webhook</Queue>",
|
||||
"<Filter><S3Key><FilterRule><Name>Prefix</Name><Value>logs/</Value></FilterRule>",
|
||||
"<FilterRule><Name>Suffix</Name><Value>.csv</Value></FilterRule></S3Key></Filter></QueueConfiguration>",
|
||||
"<TopicConfiguration><Event>s3:ObjectRemoved:*</Event><Topic>arn:rustfs:sqs:us-east-1:topic:webhook</Topic></TopicConfiguration>",
|
||||
"<CloudFunctionConfiguration><Event>s3:ObjectCreated:Put</Event>",
|
||||
"<CloudFunction>arn:rustfs:sqs:us-east-1:lambda:webhook</CloudFunction></CloudFunctionConfiguration>",
|
||||
"</NotificationConfiguration>"
|
||||
).as_bytes().to_vec();
|
||||
metadata.notification_config =
|
||||
Some(deserialize_bucket_config(&metadata.notification_config_xml).expect("parse destinations"));
|
||||
let rules = super::rules_from_metadata(&metadata).expect("compile destinations");
|
||||
assert_eq!(rules.match_rules(EventName::ObjectCreatedPut, "logs/data.csv").len(), 2);
|
||||
assert_eq!(rules.match_rules(EventName::ObjectCreatedPut, "other/data.csv").len(), 1);
|
||||
assert_eq!(rules.match_rules(EventName::ObjectCreatedPut, "logs/data.txt").len(), 1);
|
||||
let removed = rules.match_rules(EventName::ObjectRemovedDelete, "object");
|
||||
assert_eq!(removed.len(), 1);
|
||||
assert!(removed.contains(&TargetID::new("topic".into(), "webhook".into())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notification_snapshot_compilation_rejects_invalid_persisted_routes() {
|
||||
for (destination, filter) in [
|
||||
("not-an-arn", ""),
|
||||
(
|
||||
"arn:rustfs:sqs:us-east-1:inert:webhook",
|
||||
"<Filter><S3Key><FilterRule><Value>logs/</Value></FilterRule></S3Key></Filter>",
|
||||
),
|
||||
(
|
||||
"arn:rustfs:sqs:us-east-1:inert:webhook",
|
||||
concat!(
|
||||
"<Filter><S3Key><FilterRule><Name>prefix</Name><Value>logs/</Value></FilterRule>",
|
||||
"<FilterRule><Name>Prefix</Name><Value>private/</Value></FilterRule></S3Key></Filter>"
|
||||
),
|
||||
),
|
||||
] {
|
||||
let mut metadata = BucketMetadata::new("bucket");
|
||||
metadata.notification_config_xml = format!(
|
||||
"<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event>\
|
||||
<Queue>{destination}</Queue>{filter}</QueueConfiguration></NotificationConfiguration>"
|
||||
)
|
||||
.into_bytes();
|
||||
metadata.notification_config = Some(
|
||||
deserialize_bucket_config(&metadata.notification_config_xml)
|
||||
.expect("parse invalid but well-formed persisted route"),
|
||||
);
|
||||
assert!(super::rules_from_metadata(&metadata).is_none(), "invalid route must not dispatch");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rule_engine_tracks_bucket_rule_lifecycle() {
|
||||
|
||||
@@ -70,3 +70,10 @@ pub(crate) mod crate_boundary {
|
||||
save_notify_server_config_snapshot, with_notify_server_config_read_lock,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod bucket_metadata {
|
||||
pub(crate) use super::NotifyStore;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::bucket::utils::deserialize as deserialize_bucket_config;
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
pub mod init_time;
|
||||
pub mod last_minute;
|
||||
pub mod metrics;
|
||||
pub mod scan_observation;
|
||||
pub mod startup;
|
||||
pub mod usage_candidate;
|
||||
|
||||
|
||||
@@ -13,9 +13,6 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::last_minute::{AccElem, LastMinuteLatency};
|
||||
use crate::scan_observation::{
|
||||
ScannerMetadataReadGuard, ScannerMetadataReadOutcome, ScannerScanObservation, ScannerScanObservationSnapshot,
|
||||
};
|
||||
use crate::usage_candidate::{
|
||||
ScannerUsageCandidateCount, ScannerUsageCandidateKind, ScannerUsageCandidateOutcome, USAGE_CANDIDATE_COUNTER_SLOTS,
|
||||
};
|
||||
@@ -285,19 +282,6 @@ impl ScannerWorkSource {
|
||||
}
|
||||
}
|
||||
|
||||
fn from_code(code: u8) -> Option<Self> {
|
||||
match code {
|
||||
1 => Some(Self::Usage),
|
||||
2 => Some(Self::Lifecycle),
|
||||
3 => Some(Self::BucketReplication),
|
||||
4 => Some(Self::SiteReplication),
|
||||
5 => Some(Self::Heal),
|
||||
6 => Some(Self::Bitrot),
|
||||
7 => Some(Self::Alerts),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn all() -> &'static [Self] {
|
||||
&Self::ALL
|
||||
}
|
||||
@@ -978,9 +962,36 @@ impl ScannerUsagePublicationObservation {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct LastScanCycle {
|
||||
sequence: u64,
|
||||
result: u8,
|
||||
partial_reason: ScanCyclePartialReason,
|
||||
partial_source: Option<ScannerWorkSource>,
|
||||
partial_stage: Option<&'static str>,
|
||||
duration_millis: u64,
|
||||
}
|
||||
|
||||
/// Updates only this observation if a later state write fails. A supervisor
|
||||
/// cancellation leaves the already recorded partial result and budget visible.
|
||||
pub struct PartialCycleObservation<'a> {
|
||||
latest: &'a Mutex<LastScanCycle>,
|
||||
sequence: u64,
|
||||
}
|
||||
|
||||
impl PartialCycleObservation<'_> {
|
||||
pub fn observe_state_write(self, persisted: bool) {
|
||||
if !persisted {
|
||||
let mut last = self.latest.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if last.sequence == self.sequence {
|
||||
last.partial_stage = Some("cycle_state_not_durable");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Metrics {
|
||||
startup: crate::startup::ScannerStartupObservation,
|
||||
scan_observation: ScannerScanObservation,
|
||||
operations: Vec<AtomicU64>,
|
||||
latency: Vec<LockedLastMinuteLatency>,
|
||||
actions: Vec<AtomicU64>,
|
||||
@@ -1024,11 +1035,8 @@ pub struct Metrics {
|
||||
scanner_leader_lock_held: AtomicBool,
|
||||
scanner_leader_lock_last_error: RwLock<String>,
|
||||
scanner_leader_lock_last_update_unix_secs: AtomicU64,
|
||||
last_scan_cycle_result: AtomicU8,
|
||||
last_scan_cycle_partial_reason: AtomicU8,
|
||||
last_scan_cycle_partial_source: AtomicU8,
|
||||
last_scan_cycle: Mutex<LastScanCycle>,
|
||||
last_scan_cycle_end_unix_secs: AtomicU64,
|
||||
last_scan_cycle_duration_millis: AtomicU64,
|
||||
last_scan_cycle_objects_scanned: AtomicU64,
|
||||
last_scan_cycle_directories_scanned: AtomicU64,
|
||||
/// Lifetime count of object versions walked by the data scanner, counted
|
||||
@@ -1144,7 +1152,6 @@ pub struct Metrics {
|
||||
partial_scan_cycles: AtomicU64,
|
||||
}
|
||||
|
||||
const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
|
||||
const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
|
||||
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
|
||||
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
|
||||
@@ -1176,12 +1183,12 @@ impl ScanCyclePartialReason {
|
||||
}
|
||||
}
|
||||
|
||||
fn from_code(code: u8) -> Self {
|
||||
match code {
|
||||
1 => Self::Runtime,
|
||||
2 => Self::Objects,
|
||||
3 => Self::Directories,
|
||||
_ => Self::Unknown,
|
||||
fn budget_stage(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Runtime => Some("runtime_budget"),
|
||||
Self::Objects => Some("object_budget"),
|
||||
Self::Directories => Some("directory_budget"),
|
||||
Self::Unknown => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1581,6 +1588,9 @@ pub struct ScannerMetricsReport {
|
||||
pub last_cycle_partial_source: String,
|
||||
#[serde(default)]
|
||||
pub last_cycle_partial_source_code: u64,
|
||||
/// First observed blocker for this partial cycle, including the exact budget when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub last_cycle_partial_stage: Option<String>,
|
||||
pub last_cycle_duration_seconds: f64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_objects_scanned: u64,
|
||||
@@ -1635,9 +1645,6 @@ pub struct ScannerMetricsReport {
|
||||
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
|
||||
#[serde(default)]
|
||||
pub usage_freshness: ScannerUsageFreshnessSnapshot,
|
||||
/// Node-local, process-lifetime work observations. Older nodes have no evidence.
|
||||
#[serde(default)]
|
||||
pub scan_observation: ScannerScanObservationSnapshot,
|
||||
/// Process-lifetime candidate-construction results. Missing on older reports
|
||||
/// means unobserved, not accepted or healthy. Publication is tracked separately.
|
||||
#[serde(default)]
|
||||
@@ -2078,8 +2085,18 @@ pub fn emit_scan_cycle_partial_with_source(
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
) {
|
||||
global_metrics().record_scan_cycle_partial_with_source(duration, reason, source);
|
||||
emit_scan_cycle_partial_with_stage(duration, reason, source, reason.budget_stage());
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_partial_with_stage(
|
||||
duration: Duration,
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
stage: Option<&'static str>,
|
||||
) -> PartialCycleObservation<'static> {
|
||||
let observation = global_metrics().record_scan_cycle_partial_with_stage(duration, reason, source, stage);
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
|
||||
observation
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_superseded(duration: Duration) {
|
||||
@@ -2132,38 +2149,9 @@ impl Metrics {
|
||||
&self.startup
|
||||
}
|
||||
|
||||
pub fn scan_observation(&self) -> &ScannerScanObservation {
|
||||
&self.scan_observation
|
||||
}
|
||||
|
||||
/// Start after scanner node admission, before polling the disk read future.
|
||||
/// Disk-permit and RPC waits remain included; decoding is excluded. Dropping
|
||||
/// an unfinished guard observes cancellation, not physical I/O completion.
|
||||
pub fn start_scanner_metadata_read(&self) -> ScannerMetadataReadGuard<'_> {
|
||||
self.scan_observation.start_metadata_read();
|
||||
ScannerMetadataReadGuard {
|
||||
metrics: self,
|
||||
started: std::time::Instant::now(),
|
||||
outcome: ScannerMetadataReadOutcome::Cancelled,
|
||||
bytes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn finish_scanner_metadata_read(&self, outcome: ScannerMetadataReadOutcome, bytes: u64, elapsed: Duration) {
|
||||
let bytes = if outcome == ScannerMetadataReadOutcome::Success {
|
||||
bytes
|
||||
} else {
|
||||
0
|
||||
};
|
||||
self.scan_observation.finish_metadata_read(outcome, bytes, elapsed);
|
||||
self.operations[Metric::ReadMetadata as usize].fetch_add(1, Ordering::Relaxed);
|
||||
self.latency[Metric::ReadMetadata as usize].add_size(elapsed, bytes);
|
||||
}
|
||||
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
startup: crate::startup::ScannerStartupObservation::default(),
|
||||
scan_observation: ScannerScanObservation::default(),
|
||||
operations: (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
|
||||
// Each slot gets its own fresh LockedLastMinuteLatency so that
|
||||
// different metrics never accidentally share state.
|
||||
@@ -2209,11 +2197,8 @@ impl Metrics {
|
||||
scanner_leader_lock_held: AtomicBool::new(false),
|
||||
scanner_leader_lock_last_error: RwLock::new(String::new()),
|
||||
scanner_leader_lock_last_update_unix_secs: AtomicU64::new(0),
|
||||
last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
|
||||
last_scan_cycle_partial_reason: AtomicU8::new(ScanCyclePartialReason::Unknown as u8),
|
||||
last_scan_cycle_partial_source: AtomicU8::new(0),
|
||||
last_scan_cycle: Mutex::new(LastScanCycle::default()),
|
||||
last_scan_cycle_end_unix_secs: AtomicU64::new(0),
|
||||
last_scan_cycle_duration_millis: AtomicU64::new(0),
|
||||
last_scan_cycle_objects_scanned: AtomicU64::new(0),
|
||||
last_scan_cycle_directories_scanned: AtomicU64::new(0),
|
||||
lifetime_versions_scanned: AtomicU64::new(0),
|
||||
@@ -2769,8 +2754,6 @@ impl Metrics {
|
||||
/// walked the whole namespace, so an operator can tell why scoped
|
||||
/// convergence is not happening.
|
||||
pub fn record_scanner_scope_fallback(&self, reason: impl Into<String>) {
|
||||
let reason = reason.into();
|
||||
self.scan_observation.record_scope_fallback(&reason);
|
||||
self.scanner_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
|
||||
self.scanner_scope_fallback_last_unix_secs
|
||||
.store(unix_now_secs(), Ordering::Relaxed);
|
||||
@@ -2778,7 +2761,7 @@ impl Metrics {
|
||||
Ok(guard) => guard,
|
||||
Err(poisoned) => poisoned.into_inner(),
|
||||
};
|
||||
*last_reason = reason;
|
||||
*last_reason = reason.into();
|
||||
}
|
||||
|
||||
pub fn record_scanner_usage_durable_success(&self) {
|
||||
@@ -3181,36 +3164,43 @@ impl Metrics {
|
||||
self.failed_scan_cycles.fetch_add(1, Ordering::Relaxed);
|
||||
SCAN_CYCLE_RESULT_ERROR
|
||||
};
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.last_scan_cycle_result.store(result, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_superseded(&self, duration: Duration) {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.superseded_scan_cycles.fetch_add(1, Ordering::Relaxed);
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_SUPERSEDED, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result: SCAN_CYCLE_RESULT_SUPERSEDED,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_deferred(&self, duration: Duration) {
|
||||
self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result: SCAN_CYCLE_RESULT_DEFERRED,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn record_last_cycle(&self, duration: Duration, mut cycle: LastScanCycle) -> u64 {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
cycle.duration_millis = duration_millis_saturated(duration);
|
||||
let mut latest = self.last_scan_cycle.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
cycle.sequence = latest.sequence.wrapping_add(1);
|
||||
*latest = cycle;
|
||||
cycle.sequence
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
|
||||
@@ -3223,7 +3213,16 @@ impl Metrics {
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
) {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.record_scan_cycle_partial_with_stage(duration, reason, source, reason.budget_stage());
|
||||
}
|
||||
|
||||
fn record_scan_cycle_partial_with_stage(
|
||||
&self,
|
||||
duration: Duration,
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
stage: Option<&'static str>,
|
||||
) -> PartialCycleObservation<'_> {
|
||||
self.partial_scan_cycles.fetch_add(1, Ordering::Relaxed);
|
||||
match reason {
|
||||
ScanCyclePartialReason::Unknown => &self.partial_scan_cycles_unknown,
|
||||
@@ -3232,18 +3231,25 @@ impl Metrics {
|
||||
ScanCyclePartialReason::Directories => &self.partial_scan_cycles_directories,
|
||||
}
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_PARTIAL, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason.store(reason as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source
|
||||
.store(source.map(ScannerWorkSource::code).unwrap_or_default(), Ordering::Relaxed);
|
||||
if let Some(source) = source
|
||||
&& let Some(cycles) = self.partial_scan_cycles_by_source.get(source.index())
|
||||
{
|
||||
cycles.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
let sequence = self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result: SCAN_CYCLE_RESULT_PARTIAL,
|
||||
partial_reason: reason,
|
||||
partial_source: source,
|
||||
partial_stage: stage,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
PartialCycleObservation {
|
||||
latest: &self.last_scan_cycle,
|
||||
sequence,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot {
|
||||
@@ -3832,19 +3838,19 @@ impl Metrics {
|
||||
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
|
||||
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
|
||||
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
|
||||
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
|
||||
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
|
||||
m.last_cycle_result_code = last_cycle_result as u64;
|
||||
let partial_reason = ScanCyclePartialReason::from_code(self.last_scan_cycle_partial_reason.load(Ordering::Relaxed));
|
||||
m.last_cycle_partial_reason = partial_reason.as_str().to_string();
|
||||
m.last_cycle_partial_reason_code = partial_reason as u8 as u64;
|
||||
let partial_source = ScannerWorkSource::from_code(self.last_scan_cycle_partial_source.load(Ordering::Relaxed));
|
||||
m.last_cycle_partial_source = partial_source
|
||||
let last = *self.last_scan_cycle.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
m.last_cycle_result = scan_cycle_result_label(last.result).to_string();
|
||||
m.last_cycle_result_code = last.result.into();
|
||||
m.last_cycle_partial_reason = last.partial_reason.as_str().to_string();
|
||||
m.last_cycle_partial_reason_code = last.partial_reason as u64;
|
||||
m.last_cycle_partial_source = last
|
||||
.partial_source
|
||||
.map(ScannerWorkSource::as_str)
|
||||
.unwrap_or(SCAN_CYCLE_RESULT_UNKNOWN_LABEL)
|
||||
.to_string();
|
||||
m.last_cycle_partial_source_code = partial_source.map(|source| source.code().into()).unwrap_or_default();
|
||||
m.last_cycle_duration_seconds = self.last_scan_cycle_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.last_cycle_partial_source_code = last.partial_source.map(|source| source.code().into()).unwrap_or_default();
|
||||
m.last_cycle_partial_stage = last.partial_stage.map(str::to_owned);
|
||||
m.last_cycle_duration_seconds = last.duration_millis as f64 / 1000.0;
|
||||
m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
|
||||
m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
|
||||
m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
|
||||
@@ -3913,7 +3919,6 @@ impl Metrics {
|
||||
failed: self.scanner_transition_failed.load(Ordering::Relaxed),
|
||||
};
|
||||
m.usage_candidates = self.usage_candidate_counts();
|
||||
m.scan_observation = self.scan_observation.snapshot();
|
||||
let usage_save_result = ScannerUsageSaveResult::from_code(self.scanner_usage_last_save_result.load(Ordering::Relaxed));
|
||||
m.usage_freshness = ScannerUsageFreshnessSnapshot {
|
||||
dirty_pending_buckets: self.scanner_dirty_usage_pending_buckets.load(Ordering::Relaxed),
|
||||
@@ -5262,6 +5267,56 @@ mod tests {
|
||||
assert_eq!(report.partial_cycles, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delayed_partial_state_write_can_only_update_its_own_observation() {
|
||||
let metrics = Metrics::new();
|
||||
let record = || {
|
||||
metrics.record_scan_cycle_partial_with_stage(
|
||||
Duration::from_secs(1),
|
||||
ScanCyclePartialReason::Runtime,
|
||||
None,
|
||||
Some("runtime_budget"),
|
||||
)
|
||||
};
|
||||
let old = record();
|
||||
let current = record();
|
||||
assert_eq!(
|
||||
metrics.report().await.partial_cycles,
|
||||
2,
|
||||
"both stops are visible before either state write finishes"
|
||||
);
|
||||
old.observe_state_write(false);
|
||||
assert_eq!(metrics.report().await.last_cycle_partial_stage.as_deref(), Some("runtime_budget"));
|
||||
current.observe_state_write(false);
|
||||
let failed = metrics.report().await;
|
||||
assert_eq!(failed.last_cycle_partial_stage.as_deref(), Some("cycle_state_not_durable"));
|
||||
assert_eq!(failed.partial_cycles, 2);
|
||||
assert_eq!(failed.last_cycle_partial_reason, "runtime");
|
||||
let superseded = record();
|
||||
metrics.record_scan_cycle_complete(true, Duration::ZERO);
|
||||
superseded.observe_state_write(false);
|
||||
assert!(metrics.report().await.last_cycle_partial_stage.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_stage_is_reported_with_the_last_result_and_cleared_on_completion() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_cycle_partial(Duration::from_secs(1), ScanCyclePartialReason::Directories);
|
||||
let partial = serde_json::to_value(metrics.report().await).unwrap();
|
||||
assert_eq!(partial["last_cycle_partial_reason"], "directories");
|
||||
assert_eq!(partial["last_cycle_partial_stage"], "directory_budget");
|
||||
metrics.record_scan_cycle_complete(true, Duration::from_secs(1));
|
||||
let finished = serde_json::to_value(metrics.report().await).unwrap();
|
||||
assert!(
|
||||
!finished
|
||||
.as_object()
|
||||
.expect("serialized report")
|
||||
.contains_key("last_cycle_partial_stage")
|
||||
);
|
||||
let legacy: ScannerMetricsReport = serde_json::from_value(finished).expect("older reports omit the stage");
|
||||
assert_eq!(legacy.last_cycle_partial_stage, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_tracks_partial_scan_cycle_without_failed_increment() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -1,675 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Node-local, process-lifetime observations of scanner work. Counters are
|
||||
//! independently loaded, so concurrent reports need not balance exactly.
|
||||
//! These observations never grant cache reuse or publication authority.
|
||||
|
||||
use crate::metrics::Metrics;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
pub const PREFIX_REUSE_BLOCKERS: [&str; 8] = [
|
||||
"missing_producer_identity",
|
||||
"missing_durable_journal_replay",
|
||||
"restart_gap",
|
||||
"generation_gap",
|
||||
"overflow",
|
||||
"stale_ack_generation",
|
||||
"missing_cold_zero_walk_oracle",
|
||||
"distributed_without_peer_invalidation",
|
||||
];
|
||||
|
||||
const SCOPE_FALLBACK_REASONS: [&str; 6] = [
|
||||
"dirty_snapshot_incomplete",
|
||||
"dirty_generation_exhausted",
|
||||
"dirty_bucket_overflow",
|
||||
"no_listed_dirty_buckets",
|
||||
"baseline_unavailable",
|
||||
"peer_scope_unavailable",
|
||||
];
|
||||
|
||||
/// A storage read result, independent of later metadata decoding or ILM work.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerMetadataReadOutcome {
|
||||
Success,
|
||||
NotFound,
|
||||
Error,
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
/// The existing path that actually copied a cached subtree instead of walking it.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerCacheReuseKind {
|
||||
CleanPrefix,
|
||||
CoveredNamespace,
|
||||
SampledNamespace,
|
||||
ClassifiedNamespace,
|
||||
}
|
||||
|
||||
impl ScannerCacheReuseKind {
|
||||
const ALL: [Self; 4] = [
|
||||
Self::CleanPrefix,
|
||||
Self::CoveredNamespace,
|
||||
Self::SampledNamespace,
|
||||
Self::ClassifiedNamespace,
|
||||
];
|
||||
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::CleanPrefix => "clean_prefix",
|
||||
Self::CoveredNamespace => "covered_namespace",
|
||||
Self::SampledNamespace => "sampled_namespace",
|
||||
Self::ClassifiedNamespace => "classified_namespace",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Completed bucket-level reuse work, independent of durable publication.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerBucketReuseKind {
|
||||
ScopedColdBucket,
|
||||
CurrentRoot,
|
||||
}
|
||||
|
||||
impl ScannerBucketReuseKind {
|
||||
const ALL: [Self; 2] = [Self::ScopedColdBucket, Self::CurrentRoot];
|
||||
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::ScopedColdBucket => "scoped_cold_bucket",
|
||||
Self::CurrentRoot => "current_root",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A production folder gate, not a diagnostic preflight.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerCacheReuseGate {
|
||||
CleanPrefix,
|
||||
CoveredNamespace,
|
||||
NamespaceSampling,
|
||||
}
|
||||
|
||||
impl ScannerCacheReuseGate {
|
||||
const ALL: [Self; 3] = [Self::CleanPrefix, Self::CoveredNamespace, Self::NamespaceSampling];
|
||||
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::CleanPrefix => "clean_prefix",
|
||||
Self::CoveredNamespace => "covered_namespace",
|
||||
Self::NamespaceSampling => "namespace_sampling",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The first existing condition reached by a folder reuse gate.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerCacheReuseDecision {
|
||||
MissingScope,
|
||||
NoResumeSweep,
|
||||
DeepScan,
|
||||
ForwardSweep,
|
||||
ActiveReplication,
|
||||
NotRoot,
|
||||
CompactedParent,
|
||||
MissingCachedChild,
|
||||
InvalidPrefix,
|
||||
DirtyPrefix,
|
||||
ErasureHealthRequired,
|
||||
MissingCoverage,
|
||||
CachedFailures,
|
||||
UncompactedChild,
|
||||
SelectedForWalk,
|
||||
Eligible,
|
||||
}
|
||||
|
||||
impl ScannerCacheReuseDecision {
|
||||
const ALL: [Self; 16] = [
|
||||
Self::MissingScope,
|
||||
Self::NoResumeSweep,
|
||||
Self::DeepScan,
|
||||
Self::ForwardSweep,
|
||||
Self::ActiveReplication,
|
||||
Self::NotRoot,
|
||||
Self::CompactedParent,
|
||||
Self::MissingCachedChild,
|
||||
Self::InvalidPrefix,
|
||||
Self::DirtyPrefix,
|
||||
Self::ErasureHealthRequired,
|
||||
Self::MissingCoverage,
|
||||
Self::CachedFailures,
|
||||
Self::UncompactedChild,
|
||||
Self::SelectedForWalk,
|
||||
Self::Eligible,
|
||||
];
|
||||
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::MissingScope => "missing_scope",
|
||||
Self::NoResumeSweep => "no_resume_sweep",
|
||||
Self::DeepScan => "deep_scan",
|
||||
Self::ForwardSweep => "forward_sweep",
|
||||
Self::ActiveReplication => "active_replication",
|
||||
Self::NotRoot => "not_root",
|
||||
Self::CompactedParent => "compacted_parent",
|
||||
Self::MissingCachedChild => "missing_cached_child",
|
||||
Self::InvalidPrefix => "invalid_prefix",
|
||||
Self::DirtyPrefix => "dirty_prefix",
|
||||
Self::ErasureHealthRequired => "erasure_health_required",
|
||||
Self::MissingCoverage => "missing_coverage",
|
||||
Self::CachedFailures => "cached_failures",
|
||||
Self::UncompactedChild => "uncompacted_child",
|
||||
Self::SelectedForWalk => "selected_for_walk",
|
||||
Self::Eligible => "eligible",
|
||||
}
|
||||
}
|
||||
|
||||
fn not_evaluated(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::MissingScope | Self::NoResumeSweep | Self::DeepScan | Self::ForwardSweep | Self::ActiveReplication
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Scope admission and end-of-cycle evidence are separate observations. An
|
||||
/// activated completion proof does not mean this cycle used prefix reuse.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerPrefixReuseStage {
|
||||
ScopeResolution,
|
||||
CycleCompletion,
|
||||
}
|
||||
|
||||
/// The first existing baseline validation check that rejected a snapshot.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ScannerBaselineRejection {
|
||||
Missing,
|
||||
Decode,
|
||||
Incomplete,
|
||||
Partial,
|
||||
Marks,
|
||||
Epoch,
|
||||
SourceCount,
|
||||
PlanMissing,
|
||||
PlanMismatch,
|
||||
SourceIndex,
|
||||
SourceMembership,
|
||||
SourceDuplicate,
|
||||
SetIncomplete,
|
||||
SetTombstone,
|
||||
SetEpoch,
|
||||
SetCycle,
|
||||
SetPlan,
|
||||
SourceCoverage,
|
||||
AuthoritativeIdentity,
|
||||
NotNewer,
|
||||
}
|
||||
|
||||
impl ScannerBaselineRejection {
|
||||
const ALL: [Self; 20] = [
|
||||
Self::Missing,
|
||||
Self::Decode,
|
||||
Self::Incomplete,
|
||||
Self::Partial,
|
||||
Self::Marks,
|
||||
Self::Epoch,
|
||||
Self::SourceCount,
|
||||
Self::PlanMissing,
|
||||
Self::PlanMismatch,
|
||||
Self::SourceIndex,
|
||||
Self::SourceMembership,
|
||||
Self::SourceDuplicate,
|
||||
Self::SetIncomplete,
|
||||
Self::SetTombstone,
|
||||
Self::SetEpoch,
|
||||
Self::SetCycle,
|
||||
Self::SetPlan,
|
||||
Self::SourceCoverage,
|
||||
Self::AuthoritativeIdentity,
|
||||
Self::NotNewer,
|
||||
];
|
||||
|
||||
fn label(self) -> &'static str {
|
||||
match self {
|
||||
Self::Missing => "missing",
|
||||
Self::Decode => "decode",
|
||||
Self::Incomplete => "incomplete",
|
||||
Self::Partial => "partial",
|
||||
Self::Marks => "marks",
|
||||
Self::Epoch => "epoch",
|
||||
Self::SourceCount => "source_count",
|
||||
Self::PlanMissing => "plan_missing",
|
||||
Self::PlanMismatch => "plan_mismatch",
|
||||
Self::SourceIndex => "source_index",
|
||||
Self::SourceMembership => "source_membership",
|
||||
Self::SourceDuplicate => "source_duplicate",
|
||||
Self::SetIncomplete => "set_incomplete",
|
||||
Self::SetTombstone => "set_tombstone",
|
||||
Self::SetEpoch => "set_epoch",
|
||||
Self::SetCycle => "set_cycle",
|
||||
Self::SetPlan => "set_plan",
|
||||
Self::SourceCoverage => "source_coverage",
|
||||
Self::AuthoritativeIdentity => "authoritative_identity",
|
||||
Self::NotNewer => "not_newer",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ScannerMetadataReadSnapshot {
|
||||
pub attempts: u64,
|
||||
pub succeeded: u64,
|
||||
pub not_found: u64,
|
||||
pub errors: u64,
|
||||
pub cancelled: u64,
|
||||
/// Bytes returned by successful storage reads, including corrupt metadata.
|
||||
pub bytes: u64,
|
||||
/// Elapsed read-future time, including failed and cancelled reads.
|
||||
pub elapsed_nanos: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ScannerCacheReuseSnapshot {
|
||||
/// Completed subtree reuse decisions; not cache entries or objects skipped.
|
||||
pub count: u64,
|
||||
/// Time spent copying and attaching the reused subtree, excluding listing.
|
||||
pub elapsed_nanos: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ScannerBucketReuseSnapshot {
|
||||
/// Cold bucket copies or current roots successfully handed to the collector.
|
||||
pub count: u64,
|
||||
/// Cold copy/attachment time or current-root channel-send time; excludes validation.
|
||||
pub elapsed_nanos: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ScannerCacheReuseDecisionSnapshot {
|
||||
/// Policy prerequisites that stopped the gate before cache proof checks.
|
||||
pub not_evaluated: BTreeMap<String, u64>,
|
||||
/// First failed cache/work condition; later conditions are not evaluated.
|
||||
pub blocked: BTreeMap<String, u64>,
|
||||
/// Admitted candidates, which may still need EC classification before copying.
|
||||
pub eligible: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ScannerPrefixReuseSnapshot {
|
||||
pub evaluated: u64,
|
||||
pub activated: u64,
|
||||
/// All failed checks are counted; one evaluation can have several blockers.
|
||||
pub blockers: BTreeMap<String, u64>,
|
||||
}
|
||||
|
||||
/// Missing on an older node means unobserved, not zero work or healthy reuse.
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct ScannerScanObservationSnapshot {
|
||||
pub metadata_reads: ScannerMetadataReadSnapshot,
|
||||
pub cache_reuse: BTreeMap<String, ScannerCacheReuseSnapshot>,
|
||||
pub bucket_reuse: BTreeMap<String, ScannerBucketReuseSnapshot>,
|
||||
pub reuse_decisions: BTreeMap<String, ScannerCacheReuseDecisionSnapshot>,
|
||||
/// Clean root children still walked because EC scans also check health.
|
||||
pub erasure_clean_prefix_walks: u64,
|
||||
pub prefix_scope_resolution: ScannerPrefixReuseSnapshot,
|
||||
pub prefix_cycle_completion: ScannerPrefixReuseSnapshot,
|
||||
pub scope_fallbacks: BTreeMap<String, u64>,
|
||||
/// Rejections only for cycles with a baseline_unavailable scope fallback.
|
||||
pub authoritative_baseline_rejections: BTreeMap<String, u64>,
|
||||
pub observed_baseline_rejections: BTreeMap<String, u64>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ScannerScanObservation {
|
||||
metadata: [AtomicU64; 7],
|
||||
cache_count: [AtomicU64; 4],
|
||||
cache_nanos: [AtomicU64; 4],
|
||||
bucket_count: [AtomicU64; 2],
|
||||
bucket_nanos: [AtomicU64; 2],
|
||||
reuse_decisions: [[AtomicU64; ScannerCacheReuseDecision::ALL.len()]; 3],
|
||||
erasure_clean_prefix_walks: AtomicU64,
|
||||
prefix_evaluated: [AtomicU64; 2],
|
||||
prefix_activated: [AtomicU64; 2],
|
||||
prefix_blockers: [[AtomicU64; PREFIX_REUSE_BLOCKERS.len()]; 2],
|
||||
scope_fallbacks: [AtomicU64; SCOPE_FALLBACK_REASONS.len()],
|
||||
baseline_rejections: [[AtomicU64; ScannerBaselineRejection::ALL.len()]; 2],
|
||||
}
|
||||
|
||||
fn duration_nanos(duration: Duration) -> u64 {
|
||||
u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
fn labelled_counts(labels: impl IntoIterator<Item = &'static str>, counters: &[AtomicU64]) -> BTreeMap<String, u64> {
|
||||
labels
|
||||
.into_iter()
|
||||
.zip(counters)
|
||||
.map(|(label, count)| (label.to_owned(), count.load(Ordering::Relaxed)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl ScannerScanObservation {
|
||||
pub(crate) fn start_metadata_read(&self) {
|
||||
self.metadata[0].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn finish_metadata_read(&self, outcome: ScannerMetadataReadOutcome, bytes: u64, elapsed: Duration) {
|
||||
self.metadata[1 + outcome as usize].fetch_add(1, Ordering::Relaxed);
|
||||
if outcome == ScannerMetadataReadOutcome::Success {
|
||||
self.metadata[5].fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
self.metadata[6].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_cache_reuse(&self, kind: ScannerCacheReuseKind, elapsed: Duration) {
|
||||
self.cache_count[kind as usize].fetch_add(1, Ordering::Relaxed);
|
||||
self.cache_nanos[kind as usize].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records completed reuse work only; it grants no publication authority.
|
||||
pub fn record_bucket_reuse(&self, kind: ScannerBucketReuseKind, count: u64, elapsed: Duration) {
|
||||
self.bucket_count[kind as usize].fetch_add(count, Ordering::Relaxed);
|
||||
self.bucket_nanos[kind as usize].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records one first-condition decision without inspecting later prerequisites.
|
||||
pub fn record_cache_reuse_decision(&self, gate: ScannerCacheReuseGate, decision: ScannerCacheReuseDecision) {
|
||||
self.reuse_decisions[gate as usize][decision as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_erasure_clean_prefix_walk(&self) {
|
||||
self.erasure_clean_prefix_walks.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_prefix_reuse(&self, stage: ScannerPrefixReuseStage, activated: bool, blockers: &[Option<&str>; 8]) {
|
||||
let stage = stage as usize;
|
||||
self.prefix_evaluated[stage].fetch_add(1, Ordering::Relaxed);
|
||||
if activated {
|
||||
self.prefix_activated[stage].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
for blocker in blockers.iter().flatten() {
|
||||
if let Some(index) = PREFIX_REUSE_BLOCKERS.iter().position(|label| label == blocker) {
|
||||
self.prefix_blockers[stage][index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_scope_fallback(&self, reason: &str) {
|
||||
if let Some(index) = SCOPE_FALLBACK_REASONS.iter().position(|label| *label == reason) {
|
||||
self.scope_fallbacks[index].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_baseline_rejection(&self, authoritative: ScannerBaselineRejection, observed: Option<ScannerBaselineRejection>) {
|
||||
self.baseline_rejections[0][authoritative as usize].fetch_add(1, Ordering::Relaxed);
|
||||
if let Some(observed) = observed {
|
||||
self.baseline_rejections[1][observed as usize].fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot(&self) -> ScannerScanObservationSnapshot {
|
||||
let prefix = |stage: ScannerPrefixReuseStage| ScannerPrefixReuseSnapshot {
|
||||
evaluated: self.prefix_evaluated[stage as usize].load(Ordering::Relaxed),
|
||||
activated: self.prefix_activated[stage as usize].load(Ordering::Relaxed),
|
||||
blockers: labelled_counts(PREFIX_REUSE_BLOCKERS, &self.prefix_blockers[stage as usize]),
|
||||
};
|
||||
ScannerScanObservationSnapshot {
|
||||
metadata_reads: ScannerMetadataReadSnapshot {
|
||||
attempts: self.metadata[0].load(Ordering::Relaxed),
|
||||
succeeded: self.metadata[1].load(Ordering::Relaxed),
|
||||
not_found: self.metadata[2].load(Ordering::Relaxed),
|
||||
errors: self.metadata[3].load(Ordering::Relaxed),
|
||||
cancelled: self.metadata[4].load(Ordering::Relaxed),
|
||||
bytes: self.metadata[5].load(Ordering::Relaxed),
|
||||
elapsed_nanos: self.metadata[6].load(Ordering::Relaxed),
|
||||
},
|
||||
cache_reuse: ScannerCacheReuseKind::ALL
|
||||
.into_iter()
|
||||
.map(|kind| {
|
||||
(
|
||||
kind.label().to_owned(),
|
||||
ScannerCacheReuseSnapshot {
|
||||
count: self.cache_count[kind as usize].load(Ordering::Relaxed),
|
||||
elapsed_nanos: self.cache_nanos[kind as usize].load(Ordering::Relaxed),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
bucket_reuse: ScannerBucketReuseKind::ALL
|
||||
.into_iter()
|
||||
.map(|kind| {
|
||||
(
|
||||
kind.label().to_owned(),
|
||||
ScannerBucketReuseSnapshot {
|
||||
count: self.bucket_count[kind as usize].load(Ordering::Relaxed),
|
||||
elapsed_nanos: self.bucket_nanos[kind as usize].load(Ordering::Relaxed),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
reuse_decisions: ScannerCacheReuseGate::ALL
|
||||
.into_iter()
|
||||
.map(|gate| {
|
||||
let counts = &self.reuse_decisions[gate as usize];
|
||||
let decisions = |not_evaluated| {
|
||||
ScannerCacheReuseDecision::ALL
|
||||
.into_iter()
|
||||
.filter(|decision| {
|
||||
*decision != ScannerCacheReuseDecision::Eligible && decision.not_evaluated() == not_evaluated
|
||||
})
|
||||
.map(|decision| (decision.label().to_owned(), counts[decision as usize].load(Ordering::Relaxed)))
|
||||
.collect()
|
||||
};
|
||||
(
|
||||
gate.label().to_owned(),
|
||||
ScannerCacheReuseDecisionSnapshot {
|
||||
not_evaluated: decisions(true),
|
||||
blocked: decisions(false),
|
||||
eligible: counts[ScannerCacheReuseDecision::Eligible as usize].load(Ordering::Relaxed),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
erasure_clean_prefix_walks: self.erasure_clean_prefix_walks.load(Ordering::Relaxed),
|
||||
prefix_scope_resolution: prefix(ScannerPrefixReuseStage::ScopeResolution),
|
||||
prefix_cycle_completion: prefix(ScannerPrefixReuseStage::CycleCompletion),
|
||||
scope_fallbacks: labelled_counts(SCOPE_FALLBACK_REASONS, &self.scope_fallbacks),
|
||||
authoritative_baseline_rejections: labelled_counts(
|
||||
ScannerBaselineRejection::ALL.map(ScannerBaselineRejection::label),
|
||||
&self.baseline_rejections[0],
|
||||
),
|
||||
observed_baseline_rejections: labelled_counts(
|
||||
ScannerBaselineRejection::ALL.map(ScannerBaselineRejection::label),
|
||||
&self.baseline_rejections[1],
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Dropping an unfinished read records cancellation. The guard observes the
|
||||
/// read future only; it cannot cancel it or change its storage result.
|
||||
#[must_use]
|
||||
pub struct ScannerMetadataReadGuard<'a> {
|
||||
pub(crate) metrics: &'a Metrics,
|
||||
pub(crate) started: Instant,
|
||||
pub(crate) outcome: ScannerMetadataReadOutcome,
|
||||
pub(crate) bytes: u64,
|
||||
}
|
||||
|
||||
impl ScannerMetadataReadGuard<'_> {
|
||||
pub fn finish(mut self, outcome: ScannerMetadataReadOutcome, bytes: u64) {
|
||||
self.outcome = outcome;
|
||||
self.bytes = bytes;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScannerMetadataReadGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.metrics
|
||||
.finish_scanner_metadata_read(self.outcome, self.bytes, self.started.elapsed());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_read_reports_storage_results_and_existing_read_metric() {
|
||||
let metrics = Metrics::new();
|
||||
metrics
|
||||
.start_scanner_metadata_read()
|
||||
.finish(ScannerMetadataReadOutcome::Success, 127);
|
||||
metrics
|
||||
.start_scanner_metadata_read()
|
||||
.finish(ScannerMetadataReadOutcome::Success, 0);
|
||||
metrics
|
||||
.start_scanner_metadata_read()
|
||||
.finish(ScannerMetadataReadOutcome::NotFound, 0);
|
||||
metrics
|
||||
.start_scanner_metadata_read()
|
||||
.finish(ScannerMetadataReadOutcome::Error, 0);
|
||||
let report = metrics.report().await;
|
||||
let reads = &report.scan_observation.metadata_reads;
|
||||
assert_eq!(
|
||||
(reads.attempts, reads.succeeded, reads.not_found, reads.errors, reads.cancelled),
|
||||
(4, 2, 1, 1, 0)
|
||||
);
|
||||
assert_eq!(reads.bytes, 127);
|
||||
assert_eq!(report.life_time_ops["read_metadata"], 4);
|
||||
assert_eq!(report.last_minute.actions["read_metadata"].count, 4);
|
||||
assert_eq!(report.last_minute.actions["read_metadata"].bytes, 127);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_read_cancellation_balances_an_actually_polled_future_once() {
|
||||
let metrics = Arc::new(Metrics::new());
|
||||
let worker_metrics = Arc::clone(&metrics);
|
||||
let (started, admitted) = tokio::sync::oneshot::channel();
|
||||
let worker = tokio::spawn(async move {
|
||||
let read = worker_metrics.start_scanner_metadata_read();
|
||||
started.send(()).expect("observe started read");
|
||||
std::future::pending::<()>().await;
|
||||
read.finish(ScannerMetadataReadOutcome::Success, 99);
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), admitted)
|
||||
.await
|
||||
.expect("read starts within deadline")
|
||||
.expect("read started");
|
||||
let in_flight = metrics.scan_observation().snapshot().metadata_reads;
|
||||
assert_eq!((in_flight.attempts, in_flight.cancelled, in_flight.succeeded), (1, 0, 0));
|
||||
worker.abort();
|
||||
assert!(worker.await.expect_err("read task was aborted").is_cancelled());
|
||||
let report = metrics.report().await;
|
||||
let reads = &report.scan_observation.metadata_reads;
|
||||
assert_eq!((reads.attempts, reads.cancelled, reads.succeeded, reads.bytes), (1, 1, 0, 0));
|
||||
assert_eq!(report.life_time_ops["read_metadata"], 1);
|
||||
assert_eq!(report.last_minute.actions["read_metadata"].count, 1);
|
||||
assert_eq!(
|
||||
Duration::from_nanos(reads.elapsed_nanos).as_secs(),
|
||||
report.last_minute.actions["read_metadata"].acc_time
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn observation_report_keeps_stages_reasons_and_copy_costs_separate() {
|
||||
let metrics = Metrics::new();
|
||||
let observation = metrics.scan_observation();
|
||||
observation.record_cache_reuse(ScannerCacheReuseKind::CleanPrefix, Duration::from_nanos(321));
|
||||
observation.record_bucket_reuse(ScannerBucketReuseKind::ScopedColdBucket, 2, Duration::from_nanos(654));
|
||||
observation.record_bucket_reuse(ScannerBucketReuseKind::CurrentRoot, 1, Duration::from_nanos(987));
|
||||
observation.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, ScannerCacheReuseDecision::MissingScope);
|
||||
observation
|
||||
.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, ScannerCacheReuseDecision::ErasureHealthRequired);
|
||||
observation.record_cache_reuse_decision(ScannerCacheReuseGate::CoveredNamespace, ScannerCacheReuseDecision::Eligible);
|
||||
observation.record_erasure_clean_prefix_walk();
|
||||
observation.record_prefix_reuse(
|
||||
ScannerPrefixReuseStage::ScopeResolution,
|
||||
false,
|
||||
&[
|
||||
Some("missing_producer_identity"),
|
||||
Some("missing_durable_journal_replay"),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
],
|
||||
);
|
||||
observation.record_prefix_reuse(ScannerPrefixReuseStage::CycleCompletion, true, &[None; 8]);
|
||||
metrics.record_scanner_scope_fallback("baseline_unavailable");
|
||||
observation.record_baseline_rejection(ScannerBaselineRejection::Epoch, Some(ScannerBaselineRejection::Missing));
|
||||
let report = metrics.report().await;
|
||||
let snapshot = &report.scan_observation;
|
||||
assert_eq!(snapshot.cache_reuse["clean_prefix"].count, 1);
|
||||
assert_eq!(snapshot.cache_reuse["clean_prefix"].elapsed_nanos, 321);
|
||||
assert_eq!(snapshot.cache_reuse["sampled_namespace"].count, 0);
|
||||
assert_eq!(snapshot.bucket_reuse["scoped_cold_bucket"].count, 2);
|
||||
assert_eq!(snapshot.bucket_reuse["scoped_cold_bucket"].elapsed_nanos, 654);
|
||||
assert_eq!(snapshot.bucket_reuse["current_root"].count, 1);
|
||||
assert_eq!(snapshot.bucket_reuse["current_root"].elapsed_nanos, 987);
|
||||
assert_eq!(snapshot.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"], 1);
|
||||
assert_eq!(snapshot.reuse_decisions["clean_prefix"].blocked["erasure_health_required"], 1);
|
||||
assert_eq!(snapshot.reuse_decisions["covered_namespace"].eligible, 1);
|
||||
|
||||
let mut prior_observations = serde_json::to_value(snapshot).expect("encode observations");
|
||||
let fields = prior_observations.as_object_mut().expect("observation object");
|
||||
fields.remove("bucket_reuse");
|
||||
fields.remove("reuse_decisions");
|
||||
let prior: ScannerScanObservationSnapshot =
|
||||
serde_json::from_value(prior_observations).expect("read earlier observations");
|
||||
assert!(prior.bucket_reuse.is_empty());
|
||||
assert!(prior.reuse_decisions.is_empty());
|
||||
assert_eq!(prior.cache_reuse["clean_prefix"].count, 1);
|
||||
|
||||
assert_eq!(snapshot.erasure_clean_prefix_walks, 1);
|
||||
assert_eq!(
|
||||
(snapshot.prefix_scope_resolution.evaluated, snapshot.prefix_scope_resolution.activated),
|
||||
(1, 0)
|
||||
);
|
||||
assert_eq!(snapshot.prefix_scope_resolution.blockers["missing_producer_identity"], 1);
|
||||
assert_eq!(snapshot.prefix_scope_resolution.blockers["missing_durable_journal_replay"], 1);
|
||||
assert_eq!(
|
||||
(snapshot.prefix_cycle_completion.evaluated, snapshot.prefix_cycle_completion.activated),
|
||||
(1, 1)
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.scope_fallbacks["baseline_unavailable"],
|
||||
report.usage_freshness.scope_fallback_total
|
||||
);
|
||||
assert_eq!(snapshot.authoritative_baseline_rejections["epoch"], 1);
|
||||
assert_eq!(snapshot.observed_baseline_rejections["missing"], 1);
|
||||
|
||||
let mut legacy = serde_json::to_value(&report).expect("encode report");
|
||||
legacy
|
||||
.as_object_mut()
|
||||
.expect("report is an object")
|
||||
.remove("scan_observation");
|
||||
let decoded: crate::metrics::ScannerMetricsReport = serde_json::from_value(legacy).expect("read legacy report");
|
||||
assert!(decoded.scan_observation.cache_reuse.is_empty(), "legacy counters are unobserved");
|
||||
assert_eq!(decoded.scan_observation.metadata_reads.attempts, 0);
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
tempfile.workspace = true
|
||||
hotpath.workspace = true
|
||||
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
||||
rustfs-common = { workspace = true }
|
||||
@@ -108,6 +109,13 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng"] }
|
||||
bytes.workspace = true
|
||||
hex-simd.workspace = true
|
||||
|
||||
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
|
||||
rustix = { workspace = true, features = ["fs"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
same-file.workspace = true
|
||||
winapi-util.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
opentelemetry-proto.workspace = true
|
||||
@@ -119,7 +127,6 @@ rustfs-test-utils.workspace = true
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
# Test-only: pins the emitted scanner alert wire names against the canonical
|
||||
|
||||
@@ -200,36 +200,25 @@ pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Eq)]
|
||||
pub(crate) struct DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision,
|
||||
backup: Option<DataUsageCacheRevision>,
|
||||
damaged_pages: DamagedPages,
|
||||
}
|
||||
|
||||
/// In-memory provenance of the decoded cache and the revisions observed by its loader.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DataUsageCacheBaselineRevisions {
|
||||
revisions: DataUsageCacheRevisions,
|
||||
loaded_from_main: bool,
|
||||
// Admissions compare CAS tokens. A reader's local repair hint does not
|
||||
// indicate that either committed object changed while its locks were released.
|
||||
impl PartialEq for DataUsageCacheRevisions {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.main == other.main && self.backup == other.backup
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageCacheBaselineRevisions {
|
||||
pub(crate) fn revisions(&self) -> &DataUsageCacheRevisions {
|
||||
&self.revisions
|
||||
}
|
||||
|
||||
/// A backup may catch up to an unchanged primary without changing the decoded baseline.
|
||||
/// Both reads must have decoded that primary; an ETag on a corrupt primary is insufficient.
|
||||
pub(crate) fn permits_unchanged_baseline_since(&self, initial: &Self) -> bool {
|
||||
if self.revisions == initial.revisions {
|
||||
return true;
|
||||
}
|
||||
self.loaded_from_main
|
||||
&& initial.loaded_from_main
|
||||
&& matches!(&initial.revisions.main, DataUsageCacheRevision::Etag(etag) if !etag.is_empty())
|
||||
&& self.revisions.main == initial.revisions.main
|
||||
&& self.revisions.backup.as_ref() == Some(&self.revisions.main)
|
||||
}
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
struct DamagedPages {
|
||||
main: bool,
|
||||
backup: bool,
|
||||
}
|
||||
|
||||
pub static DATA_USAGE_BUCKET: LazyLock<String> =
|
||||
@@ -602,10 +591,31 @@ pub(crate) struct DataUsageDurableProgress {
|
||||
identity: Option<DataUsageScanIdentity>,
|
||||
started_plan: Option<DataUsageScanPlanDigest>,
|
||||
frontier: Option<String>,
|
||||
dormant: HashMap<String, String>,
|
||||
raw: HashMap<String, crate::raw_page_index::RawEnumerationDurableProgress>,
|
||||
compacted: Vec<crate::scanner_folder::DirectoryProgress>,
|
||||
listings: Vec<crate::scanner_folder::DirectoryReadProgress>,
|
||||
active_source: Option<String>,
|
||||
}
|
||||
|
||||
impl DataUsageDurableProgress {
|
||||
fn source_frontier(&self, source: Option<&str>) -> Option<&str> {
|
||||
if source == self.active_source.as_deref() {
|
||||
self.frontier.as_deref()
|
||||
} else {
|
||||
source.and_then(|source| self.dormant.get(source)).map(String::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
fn covers_on(&self, source: Option<&str>, path: &str) -> bool {
|
||||
self.source_frontier(source)
|
||||
.is_some_and(|frontier| scan_frontier_covers_path(path, frontier))
|
||||
|| self
|
||||
.compacted
|
||||
.iter()
|
||||
.any(|done| Some(done.physical.endpoint.as_str()) == source && done.covers(path))
|
||||
}
|
||||
|
||||
pub(crate) fn advanced_from(&self, previous: &Self) -> bool {
|
||||
if self.bucket != previous.bucket
|
||||
|| self.source.is_none()
|
||||
@@ -617,31 +627,69 @@ impl DataUsageDurableProgress {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut advanced = match (&previous.frontier, &self.frontier) {
|
||||
(Some(before), Some(after)) => {
|
||||
if !scan_frontier_covers_path(before, after) {
|
||||
return false;
|
||||
let active = self.active_source.as_deref();
|
||||
let old = previous.active_source.as_deref();
|
||||
let switched = old.is_some() && active != old;
|
||||
// A frontier can move from active to dormant without earning a turn.
|
||||
// Retention and forward movement are compared only on the same source.
|
||||
if previous.frontier.as_deref().is_some_and(|path| !self.covers_on(old, path))
|
||||
|| previous
|
||||
.dormant
|
||||
.iter()
|
||||
.any(|(source, path)| !self.covers_on(Some(source), path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut advanced = self.frontier.as_deref().is_some_and(|now| {
|
||||
previous
|
||||
.source_frontier(active)
|
||||
.is_none_or(|before| before != now && scan_frontier_covers_path(before, now))
|
||||
});
|
||||
if switched {
|
||||
// Legacy raw-name pages are local to their writer. The physical
|
||||
// root position must still exist for that writer across rotation.
|
||||
if !previous.raw.is_empty()
|
||||
&& !previous.listings.iter().any(|before| {
|
||||
before.parent == self.bucket
|
||||
&& Some(before.physical.endpoint.as_str()) == old
|
||||
&& self.listings.iter().any(|after| after.extends(before))
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
for (parent, before) in &previous.raw {
|
||||
match self.raw.get(parent) {
|
||||
Some(after) if after.extends(before) => {}
|
||||
None if self.covers_on(active, parent) => {}
|
||||
_ => return false,
|
||||
}
|
||||
before != after
|
||||
}
|
||||
(Some(_), None) => return false,
|
||||
(None, Some(_)) => true,
|
||||
(None, None) => false,
|
||||
};
|
||||
for (parent, before) in &previous.raw {
|
||||
match self.raw.get(parent) {
|
||||
Some(after) if after.extends(before) => {}
|
||||
None if self
|
||||
.frontier
|
||||
.as_ref()
|
||||
.is_some_and(|frontier| scan_frontier_covers_path(parent, frontier)) => {}
|
||||
_ => return false,
|
||||
advanced |= self.raw.iter().any(|(parent, after)| {
|
||||
after.committed_entries() > previous.raw.get(parent).map_or(0, |before| before.committed_entries())
|
||||
});
|
||||
}
|
||||
for before in &previous.listings {
|
||||
if !self.listings.iter().any(|after| after.extends(before))
|
||||
&& !self.covers_on(Some(&before.physical.endpoint), &before.parent)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (parent, after) in &self.raw {
|
||||
let before = previous.raw.get(parent).map_or(0, |value| value.committed_entries());
|
||||
advanced |= after.committed_entries() > before;
|
||||
for before in &previous.compacted {
|
||||
if !self.compacted.iter().any(|after| after.extends(before))
|
||||
&& !self.covers_on(Some(&before.physical.endpoint), &before.parent)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
advanced |= self.listings.iter().any(|after| {
|
||||
Some(after.physical.endpoint.as_str()) == active && !previous.listings.iter().any(|before| before.extends(after))
|
||||
});
|
||||
advanced |= self.compacted.iter().any(|after| {
|
||||
Some(after.physical.endpoint.as_str()) == active && !previous.compacted.iter().any(|before| before.extends(after))
|
||||
});
|
||||
|
||||
advanced
|
||||
}
|
||||
}
|
||||
@@ -931,6 +979,14 @@ pub struct DataUsageCacheInfo {
|
||||
pub scan_raw_enumeration_cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scan_raw_enumeration_page_index: Option<RawEnumerationPageIndex>,
|
||||
/// Source-bound native listing positions and fully committed compacted work.
|
||||
#[serde(default)]
|
||||
pub scan_directory_work: Vec<crate::scanner_folder::DirectoryCheckpoint>,
|
||||
/// Physical source of the regular linked tree and its frontier.
|
||||
#[serde(default)]
|
||||
pub scan_directory_source: Option<String>,
|
||||
#[serde(default)]
|
||||
pub scan_source_trees: std::collections::BTreeMap<String, Arc<source_tree::SourceTree>>,
|
||||
/// Indexes of the other directories on the active walk path. A restart
|
||||
/// must replay every wide ancestor before it reaches the selected one, and
|
||||
/// recharging those replays would stop a fixed budget from ever advancing.
|
||||
@@ -1020,6 +1076,9 @@ impl Serialize for DataUsageCacheInfo {
|
||||
+ usize::from(self.scan_raw_enumeration_cursor.is_some())
|
||||
+ usize::from(self.scan_raw_enumeration_page_index.is_some())
|
||||
+ usize::from(!self.scan_raw_enumeration_path_page_indexes.is_empty())
|
||||
+ usize::from(!self.scan_directory_work.is_empty())
|
||||
+ usize::from(self.scan_directory_source.is_some())
|
||||
+ usize::from(!self.scan_source_trees.is_empty())
|
||||
+ usize::from(self.scan_identity.is_some())
|
||||
+ usize::from(self.scan_progress.is_some())
|
||||
+ usize::from(self.scan_usage_watch.is_some())
|
||||
@@ -1055,6 +1114,15 @@ impl Serialize for DataUsageCacheInfo {
|
||||
if let Some(index) = &self.scan_raw_enumeration_page_index {
|
||||
state.serialize_entry("scan_raw_enumeration_page_index", index)?;
|
||||
}
|
||||
if !self.scan_source_trees.is_empty() {
|
||||
state.serialize_entry("scan_source_trees", &self.scan_source_trees)?;
|
||||
}
|
||||
if let Some(source) = &self.scan_directory_source {
|
||||
state.serialize_entry("scan_directory_source", source)?;
|
||||
}
|
||||
if !self.scan_directory_work.is_empty() {
|
||||
state.serialize_entry("scan_directory_work", &self.scan_directory_work)?;
|
||||
}
|
||||
if !self.scan_raw_enumeration_path_page_indexes.is_empty() {
|
||||
state.serialize_entry("scan_raw_enumeration_path_page_indexes", &self.scan_raw_enumeration_path_page_indexes)?;
|
||||
}
|
||||
@@ -1262,6 +1330,25 @@ impl DataUsageCache {
|
||||
identity: self.info.scan_identity,
|
||||
started_plan: self.info.scan_progress.map(|progress| progress.started_plan),
|
||||
frontier: self.validated_scan_frontier().map(str::to_owned),
|
||||
dormant: self
|
||||
.info
|
||||
.scan_source_trees
|
||||
.iter()
|
||||
.filter_map(|(source, tree)| tree.frontier(&self.info).map(|last| (source.clone(), last.to_owned())))
|
||||
.collect(),
|
||||
active_source: self.info.scan_directory_source.clone(),
|
||||
listings: self
|
||||
.info
|
||||
.scan_directory_work
|
||||
.iter()
|
||||
.filter_map(|d| d.read_progress(&self.info))
|
||||
.collect(),
|
||||
compacted: self
|
||||
.info
|
||||
.scan_directory_work
|
||||
.iter()
|
||||
.filter_map(|d| d.progress(&self.info))
|
||||
.collect(),
|
||||
raw: self
|
||||
.info
|
||||
.scan_raw_enumeration_page_index
|
||||
@@ -1275,7 +1362,8 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub(crate) fn has_scan_position(&self) -> bool {
|
||||
self.info.scan_resume_after.is_some()
|
||||
!self.info.scan_directory_work.is_empty()
|
||||
|| self.info.scan_resume_after.is_some()
|
||||
|| self.info.scan_checkpoint.is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some()
|
||||
|| self.info.scan_raw_enumeration_page_index.is_some()
|
||||
@@ -1345,7 +1433,8 @@ impl DataUsageCache {
|
||||
&& self.info.scan_identity == Some(identity)
|
||||
&& (self.validated_scan_frontier().is_some()
|
||||
|| self.validated_raw_enumeration_cursor().is_some()
|
||||
|| self.validated_raw_enumeration_page_index().is_some());
|
||||
|| self.validated_raw_enumeration_page_index().is_some()
|
||||
|| self.info.scan_directory_work.iter().any(|d| d.is_bound_to(&self.info)));
|
||||
let handoff_frontier = cross_epoch_checkpoint
|
||||
.then(|| self.validated_scan_frontier().map(clone_cache_value))
|
||||
.flatten();
|
||||
@@ -1430,7 +1519,10 @@ impl DataUsageCache {
|
||||
.take(MAX_DATA_USAGE_CACHE_DEPTH)
|
||||
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
|
||||
.collect();
|
||||
let directory_work = std::mem::take(&mut self.info.scan_directory_work);
|
||||
self.info.scan_directory_work = directory_work.into_iter().filter(|d| d.is_bound_to(&self.info)).collect();
|
||||
let cursor_is_valid = !self.has_scan_position()
|
||||
|| !self.info.scan_directory_work.is_empty()
|
||||
|| self.validated_scan_frontier().is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some()
|
||||
|| self.info.scan_raw_enumeration_page_index.is_some();
|
||||
@@ -1473,6 +1565,9 @@ impl DataUsageCache {
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
self.info.scan_directory_work.clear();
|
||||
self.info.scan_directory_source = None;
|
||||
self.info.scan_source_trees.clear();
|
||||
self.info.scan_coverage_receipt = None;
|
||||
}
|
||||
// Old readers do not understand coverage sweeps. An absent plan makes
|
||||
@@ -2218,6 +2313,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
mod persistence;
|
||||
mod source_tree;
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
struct Inner {
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod pages;
|
||||
mod scan_load;
|
||||
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use std::sync::Arc;
|
||||
@@ -70,7 +73,7 @@ struct DataUsageCacheLoadResult {
|
||||
cache: DataUsageCache,
|
||||
main_revision: DataUsageCacheRevision,
|
||||
backup_revision: Option<DataUsageCacheRevision>,
|
||||
loaded_from_main: bool,
|
||||
damaged_pages: DamagedPages,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -107,58 +110,55 @@ impl DataUsageCache {
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
) -> StorageResult<DataUsageCacheRevisions> {
|
||||
self.load_with_baseline_revisions(store, name)
|
||||
.await
|
||||
.map(|baseline| baseline.revisions)
|
||||
}
|
||||
|
||||
/// Loads cache revisions with provenance for unchanged-primary backup reconciliation.
|
||||
/// Missing or rebuilt caches and a decoded backup do not establish a decoded primary.
|
||||
pub(crate) async fn load_with_baseline_revisions<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
) -> StorageResult<DataUsageCacheBaselineRevisions> {
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let loaded = Self::load_cache(store.clone(), name).await?;
|
||||
let backup = match loaded.backup_revision {
|
||||
Some(revision) => Some(revision),
|
||||
None => match read_config_revision(store, &backup_path).await {
|
||||
Ok(revision) => Some(revision),
|
||||
Err(err) => {
|
||||
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
|
||||
debug!(
|
||||
target: "rustfs::scanner::data_usage",
|
||||
event = EVENT_SCANNER_CACHE_LOAD_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_CACHE,
|
||||
cache_name = %name,
|
||||
backup_path = %backup_path,
|
||||
state = "backup_revision_unavailable",
|
||||
error = %err,
|
||||
"Scanner cache backup revision lookup failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
None => Self::backup_revision(store, name).await,
|
||||
};
|
||||
let main = loaded.main_revision;
|
||||
*self = loaded.cache;
|
||||
|
||||
Ok(DataUsageCacheBaselineRevisions {
|
||||
revisions: DataUsageCacheRevisions { main, backup },
|
||||
loaded_from_main: loaded.loaded_from_main,
|
||||
Ok(DataUsageCacheRevisions {
|
||||
main,
|
||||
backup,
|
||||
damaged_pages: loaded.damaged_pages,
|
||||
})
|
||||
}
|
||||
|
||||
async fn backup_revision<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> Option<DataUsageCacheRevision> {
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
match read_config_revision(store, &backup_path).await {
|
||||
Ok(revision) => Some(revision),
|
||||
Err(err) => {
|
||||
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
|
||||
debug!(
|
||||
target: "rustfs::scanner::data_usage",
|
||||
event = EVENT_SCANNER_CACHE_LOAD_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_CACHE,
|
||||
cache_name = %name,
|
||||
backup_path = %backup_path,
|
||||
state = "backup_revision_unavailable",
|
||||
error = %err,
|
||||
"Scanner cache backup revision lookup failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn read_revisions<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheRevisions> {
|
||||
let main_path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let main = read_config_revision(store.clone(), &main_path).await?;
|
||||
let backup = Some(read_config_revision(store, &backup_path).await?);
|
||||
Ok(DataUsageCacheRevisions { main, backup })
|
||||
Ok(DataUsageCacheRevisions {
|
||||
main,
|
||||
backup,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_cache<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheLoadResult> {
|
||||
@@ -172,13 +172,11 @@ impl DataUsageCache {
|
||||
cache,
|
||||
revision: Some(main_revision),
|
||||
} => {
|
||||
// Legacy-location fallback carries Missing for the primary path.
|
||||
let loaded_from_main = matches!(&main_revision, DataUsageCacheRevision::Etag(_));
|
||||
return Ok(DataUsageCacheLoadResult {
|
||||
cache: *cache,
|
||||
main_revision,
|
||||
backup_revision: None,
|
||||
loaded_from_main,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
});
|
||||
}
|
||||
DataUsageCacheLoadAttempt::Loaded { revision: None, .. } => {
|
||||
@@ -203,6 +201,11 @@ impl DataUsageCache {
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_attempt = Self::try_load_inner(store.clone(), &backup_name, Duration::from_secs(30)).await?;
|
||||
let backup_revision = backup_attempt.revision();
|
||||
if backup_revision.is_some()
|
||||
&& let Some(DataUsageCacheRevision::Etag(etag)) = &main_revision
|
||||
{
|
||||
scan_load::PendingBackup::finish(&store, name, etag);
|
||||
}
|
||||
let backup_state = match backup_attempt {
|
||||
DataUsageCacheLoadAttempt::Loaded {
|
||||
cache,
|
||||
@@ -216,7 +219,10 @@ impl DataUsageCache {
|
||||
cache: *cache,
|
||||
main_revision,
|
||||
backup_revision: Some(backup_revision),
|
||||
loaded_from_main: false,
|
||||
damaged_pages: DamagedPages {
|
||||
main: main_state == DataUsageCacheLoadState::Corrupt,
|
||||
backup: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
DataUsageCacheLoadState::Loaded
|
||||
@@ -240,7 +246,10 @@ impl DataUsageCache {
|
||||
main_revision: main_revision
|
||||
.ok_or_else(|| Error::other(format!("scanner cache missing state has no revision: {name}")))?,
|
||||
backup_revision,
|
||||
loaded_from_main: false,
|
||||
damaged_pages: DamagedPages {
|
||||
main: main_state == DataUsageCacheLoadState::Corrupt,
|
||||
backup: backup_state == DataUsageCacheLoadState::Corrupt,
|
||||
},
|
||||
});
|
||||
}
|
||||
(DataUsageCacheLoadState::Corrupt, DataUsageCacheLoadState::Missing)
|
||||
@@ -260,7 +269,10 @@ impl DataUsageCache {
|
||||
main_revision: main_revision
|
||||
.ok_or_else(|| Error::other(format!("scanner cache corrupt state has no revision: {name}")))?,
|
||||
backup_revision,
|
||||
loaded_from_main: false,
|
||||
damaged_pages: DamagedPages {
|
||||
main: main_state == DataUsageCacheLoadState::Corrupt,
|
||||
backup: backup_state == DataUsageCacheLoadState::Corrupt,
|
||||
},
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
@@ -316,10 +328,35 @@ impl DataUsageCache {
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.cloned()
|
||||
.map(DataUsageCacheRevision::Etag);
|
||||
if let Some(DataUsageCacheRevision::Etag(etag)) = &revision
|
||||
&& scan_load::PendingBackup::needs_backup(&store, load_name, etag)
|
||||
{
|
||||
return Ok((None, revision, true));
|
||||
}
|
||||
match reader.read_all().await {
|
||||
Ok(data) => match DataUsageCache::unmarshal(&data) {
|
||||
Ok(data) => match Box::pin(pages::decode(store.clone(), load_name, &data)).await {
|
||||
Ok(cache) => Ok((Some(cache), revision, false)),
|
||||
Err(_) => Ok((None, revision, true)),
|
||||
Err(err)
|
||||
if matches!(err, StorageError::Io(ref io) if io.kind() == std::io::ErrorKind::InvalidData)
|
||||
|| matches!(err, StorageError::FileNotFound | StorageError::ObjectNotFound(..))
|
||||
|| (pages::is_manifest(&data) && pages::unreadable_page(&err)) =>
|
||||
{
|
||||
// A concurrent manifest commit can retire the generation
|
||||
// we were reading. Retry the current root before treating
|
||||
// missing pages as corruption or falling back to a backup.
|
||||
if pages::is_manifest(&data) {
|
||||
if Some(read_config_revision(store.clone(), &path).await?) != revision {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
if !load_name.ends_with(".bkp")
|
||||
&& let Some(DataUsageCacheRevision::Etag(etag)) = &revision
|
||||
{
|
||||
scan_load::PendingBackup::begin(&store, load_name, etag);
|
||||
}
|
||||
}
|
||||
Ok((None, revision, true))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
Err(e) => {
|
||||
// Read error
|
||||
@@ -665,20 +702,44 @@ impl DataUsageCache {
|
||||
expected_epoch: Option<u64>,
|
||||
) -> StorageResult<()> {
|
||||
let serialize_started = Instant::now();
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
// Primary, backup, and retries read the same immutable encoded body.
|
||||
let buf = Bytes::from(buf);
|
||||
counter!("rustfs_scanner_cache_serialized_bytes_total").increment(u64::try_from(buf.len()).unwrap_or(u64::MAX));
|
||||
let paged = matches!((revisions, expected_epoch), (Some(_), Some(_))) && pages::eligible(self, name);
|
||||
let encoded = if paged { Some(pages::encode(self)?) } else { None };
|
||||
let inline = if paged {
|
||||
Bytes::new()
|
||||
} else {
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
Bytes::from(buf)
|
||||
};
|
||||
counter!("rustfs_scanner_cache_serialized_bytes_total").increment(
|
||||
encoded
|
||||
.as_ref()
|
||||
.map_or_else(|| u64::try_from(inline.len()).unwrap_or(u64::MAX), pages::EncodedPages::length),
|
||||
);
|
||||
histogram!("rustfs_scanner_cache_serialize_seconds").record(serialize_started.elapsed().as_secs_f64());
|
||||
let timeout_duration = Self::cache_save_timeout();
|
||||
|
||||
let (primary_buf, primary_retired) = match (&encoded, revisions, expected_epoch) {
|
||||
(Some(encoded), Some(revisions), Some(epoch)) => {
|
||||
Box::pin(pages::prepare(
|
||||
self,
|
||||
store.clone(),
|
||||
name,
|
||||
&revisions.main,
|
||||
encoded,
|
||||
epoch,
|
||||
revisions.damaged_pages.main,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
_ => (inline.clone(), None),
|
||||
};
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let primary_save_started = Instant::now();
|
||||
Self::save_path_with_retry(
|
||||
store.clone(),
|
||||
&path,
|
||||
&buf,
|
||||
&primary_buf,
|
||||
timeout_duration,
|
||||
DATA_USAGE_CACHE_SAVE_RETRIES,
|
||||
revisions.map(|revisions| revisions.main.clone()),
|
||||
@@ -686,6 +747,9 @@ impl DataUsageCache {
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let (Some(retired), Some(epoch)) = (primary_retired, expected_epoch) {
|
||||
let _ = pages::reclaim(store.clone(), name, retired, epoch).await;
|
||||
}
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let backup_timeout_duration = Self::backup_cache_save_timeout(timeout_duration, primary_save_started.elapsed());
|
||||
@@ -693,17 +757,38 @@ impl DataUsageCache {
|
||||
if revisions.is_some() && backup_revision.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(e) = Self::save_path_with_retry(
|
||||
store,
|
||||
&backup_path,
|
||||
&buf,
|
||||
backup_timeout_duration,
|
||||
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
|
||||
backup_revision,
|
||||
expected_epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let save_backup = async {
|
||||
let (backup_buf, retired) = match (&encoded, &backup_revision, expected_epoch) {
|
||||
(Some(encoded), Some(revision), Some(epoch)) => {
|
||||
Box::pin(pages::prepare(
|
||||
self,
|
||||
store.clone(),
|
||||
&backup_name,
|
||||
revision,
|
||||
encoded,
|
||||
epoch,
|
||||
revisions.is_some_and(|revisions| revisions.damaged_pages.backup),
|
||||
))
|
||||
.await?
|
||||
}
|
||||
_ => (inline, None),
|
||||
};
|
||||
Self::save_path_with_retry(
|
||||
store.clone(),
|
||||
&backup_path,
|
||||
&backup_buf,
|
||||
backup_timeout_duration,
|
||||
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
|
||||
backup_revision,
|
||||
expected_epoch,
|
||||
)
|
||||
.await?;
|
||||
if let (Some(bundle), Some(epoch)) = (retired, expected_epoch) {
|
||||
let _ = pages::reclaim(store, &backup_name, bundle, epoch).await;
|
||||
}
|
||||
Ok::<_, StorageError>(())
|
||||
};
|
||||
if let Err(e) = save_backup.await {
|
||||
warn!(
|
||||
target: "rustfs::scanner::data_usage",
|
||||
event = EVENT_SCANNER_CACHE_SAVE_STATE,
|
||||
|
||||
@@ -0,0 +1,579 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
mod restore;
|
||||
mod save;
|
||||
mod staging;
|
||||
|
||||
use super::*;
|
||||
use futures::future::try_join_all;
|
||||
use sha2::Digest as _;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::io::Write;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const PAGE_MIN: usize = 8 * 1024;
|
||||
const PAGE_MAX: usize = 64 * 1024;
|
||||
const FANOUT: usize = 128;
|
||||
const IO_CONCURRENCY: usize = 4;
|
||||
const PREFIX: &[u8] = b"\x83\xa7pagesV1";
|
||||
type Digest = [u8; 32];
|
||||
|
||||
/// Older readers see the committed compact bucket or an incomplete empty tree.
|
||||
#[derive(Deserialize)]
|
||||
struct PageEnvelope {
|
||||
#[serde(rename = "pagesV1")]
|
||||
header: Header,
|
||||
info: DataUsageCacheInfo,
|
||||
#[serde(rename = "cache")]
|
||||
_cache: serde::de::IgnoredAny,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct Header {
|
||||
body: Payload,
|
||||
retired: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
enum Payload {
|
||||
Inline,
|
||||
Pages(Manifest),
|
||||
}
|
||||
|
||||
impl Header {
|
||||
fn manifest(&self) -> Option<&Manifest> {
|
||||
match &self.body {
|
||||
Payload::Pages(manifest) => Some(manifest),
|
||||
Payload::Inline => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
struct Manifest {
|
||||
bundle: uuid::Uuid,
|
||||
finished: bool,
|
||||
cycle: u64,
|
||||
length: u64,
|
||||
level: u8,
|
||||
root: Digest,
|
||||
#[serde(default)]
|
||||
bundle_bytes: u64,
|
||||
#[serde(default)]
|
||||
bundle_commits: u16,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Archive {
|
||||
blocks: BTreeMap<Digest, Bytes>,
|
||||
leaves: Vec<Digest>,
|
||||
buffer: Vec<u8>,
|
||||
rolling: u64,
|
||||
length: u64,
|
||||
}
|
||||
|
||||
fn invalid(message: &str) -> Error {
|
||||
Error::Io(std::io::Error::new(std::io::ErrorKind::InvalidData, message.to_owned()))
|
||||
}
|
||||
|
||||
pub(super) fn unreadable_page(error: &StorageError) -> bool {
|
||||
matches!(error, StorageError::FileCorrupt)
|
||||
|| matches!(error, StorageError::Io(io) if matches!(io.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof))
|
||||
}
|
||||
|
||||
fn digest(data: &[u8]) -> Digest {
|
||||
Sha256::digest(data).into()
|
||||
}
|
||||
|
||||
fn block_path(name: &str, bundle: uuid::Uuid, block: &Digest) -> String {
|
||||
let hash = hex_simd::encode_to_string(block, hex_simd::AsciiCase::Lower);
|
||||
format!("{BUCKET_META_PREFIX}/{name}.pages/{bundle}/{hash}")
|
||||
}
|
||||
|
||||
fn bucket_cache_name(name: &str) -> Option<&str> {
|
||||
let (bucket, leaf) = name.split_once('/')?;
|
||||
(!crate::is_reserved_or_invalid_bucket(bucket, true) && leaf == DATA_USAGE_CACHE_NAME).then_some(name)
|
||||
}
|
||||
|
||||
// The high bits of a multiplicative hash of each byte give a rolling
|
||||
// content-defined boundary. Insertion before a sealed run need not shift the
|
||||
// offsets of every following page. Page length is bounded for every input.
|
||||
const fn byte_gear(byte: u8) -> u64 {
|
||||
let mut x = (byte as u64).wrapping_add(0x9e37_79b9_7f4a_7c15);
|
||||
x = (x ^ (x >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
|
||||
x = (x ^ (x >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
|
||||
x ^ (x >> 31)
|
||||
}
|
||||
|
||||
const GEAR: [u64; 256] = {
|
||||
let mut table = [0; 256];
|
||||
let mut byte = 0u8;
|
||||
loop {
|
||||
table[byte as usize] = byte_gear(byte);
|
||||
if byte == u8::MAX {
|
||||
break;
|
||||
}
|
||||
byte += 1;
|
||||
}
|
||||
table
|
||||
};
|
||||
|
||||
impl Archive {
|
||||
fn intern(&mut self, bytes: Vec<u8>) -> Digest {
|
||||
let key = digest(&bytes);
|
||||
self.blocks.entry(key).or_insert_with(|| Bytes::from(bytes));
|
||||
key
|
||||
}
|
||||
|
||||
fn seal(&mut self) {
|
||||
if !self.buffer.is_empty() {
|
||||
let bytes = std::mem::take(&mut self.buffer);
|
||||
let key = self.intern(bytes);
|
||||
self.leaves.push(key);
|
||||
self.rolling = 0;
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&mut self, bundle: uuid::Uuid) -> StorageResult<Manifest> {
|
||||
self.seal();
|
||||
let mut children = std::mem::take(&mut self.leaves);
|
||||
let mut level = 0;
|
||||
while children.len() > 1 {
|
||||
children = children
|
||||
.chunks(FANOUT)
|
||||
.map(|chunk| {
|
||||
let bytes = chunk.iter().flatten().copied().collect();
|
||||
self.intern(bytes)
|
||||
})
|
||||
.collect();
|
||||
level += 1;
|
||||
}
|
||||
Ok(Manifest {
|
||||
bundle,
|
||||
finished: false,
|
||||
cycle: 0,
|
||||
length: self.length,
|
||||
level,
|
||||
root: children.pop().ok_or_else(|| invalid("empty scanner checkpoint"))?,
|
||||
bundle_bytes: 0,
|
||||
bundle_commits: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Archive {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.length = self
|
||||
.length
|
||||
.checked_add(u64::try_from(bytes.len()).map_err(std::io::Error::other)?)
|
||||
.ok_or_else(|| std::io::Error::other("scanner checkpoint is too large"))?;
|
||||
let mut start = 0;
|
||||
for (offset, byte) in bytes.iter().enumerate() {
|
||||
self.rolling = self.rolling.wrapping_shl(1).wrapping_add(GEAR[usize::from(*byte)]);
|
||||
let end = offset + 1;
|
||||
let len = self.buffer.len() + (end - start);
|
||||
if len == PAGE_MAX || (len >= PAGE_MIN && self.rolling & 0x3fff == 0) {
|
||||
self.buffer.extend_from_slice(&bytes[start..end]);
|
||||
self.seal();
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
self.buffer.extend_from_slice(&bytes[start..]);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct EncodedPages {
|
||||
archive: Archive,
|
||||
root: Manifest,
|
||||
inline: Option<Bytes>,
|
||||
}
|
||||
|
||||
impl EncodedPages {
|
||||
pub(super) fn length(&self) -> u64 {
|
||||
self.archive.length
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn encode(cache: &DataUsageCache) -> StorageResult<EncodedPages> {
|
||||
let mut archive = Archive::default();
|
||||
archive.write_all(&[0x92])?;
|
||||
cache.info.serialize(&mut rmp_serde::Serializer::new(&mut archive))?;
|
||||
archive.seal();
|
||||
let count = u32::try_from(cache.cache.len()).map_err(|_| invalid("too many scanner checkpoint entries"))?;
|
||||
// A wide root's growing child set must not shift every subtree page.
|
||||
let mut entries: Vec<_> = cache.cache.iter().collect();
|
||||
entries.sort_unstable_by(|a, b| a.0.cmp(b.0));
|
||||
let mut header = Vec::new();
|
||||
if count < 16 {
|
||||
header.push(0x80 | u8::try_from(count).map_err(|_| invalid("invalid fixed map length"))?);
|
||||
} else if let Ok(n) = u16::try_from(count) {
|
||||
header.push(0xde);
|
||||
header.extend_from_slice(&n.to_be_bytes());
|
||||
} else {
|
||||
header.push(0xdf);
|
||||
header.extend_from_slice(&count.to_be_bytes());
|
||||
}
|
||||
archive.write_all(&header)?;
|
||||
for (name, entry) in entries {
|
||||
let mut serializer = rmp_serde::Serializer::new(&mut archive);
|
||||
name.serialize(&mut serializer)?;
|
||||
entry.ordered_encoding().serialize(&mut serializer)?;
|
||||
if name == &cache.info.name {
|
||||
archive.seal();
|
||||
}
|
||||
}
|
||||
archive.seal();
|
||||
let inline = if archive.length <= PAGE_MIN as u64 {
|
||||
let mut buf = Vec::new();
|
||||
for key in &archive.leaves {
|
||||
buf.extend_from_slice(
|
||||
archive
|
||||
.blocks
|
||||
.get(key)
|
||||
.ok_or_else(|| invalid("missing encoded scanner page"))?,
|
||||
);
|
||||
}
|
||||
Some(Bytes::from(buf))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let root = archive.finish(uuid::Uuid::nil())?;
|
||||
Ok(EncodedPages { archive, root, inline })
|
||||
}
|
||||
|
||||
pub(super) fn is_manifest(bytes: &[u8]) -> bool {
|
||||
bytes.starts_with(PREFIX)
|
||||
}
|
||||
|
||||
fn read_envelope(bytes: &[u8], bucket: &str) -> StorageResult<PageEnvelope> {
|
||||
let head: PageEnvelope = rmp_serde::from_slice(bytes).map_err(|_| invalid("invalid scanner checkpoint manifest"))?;
|
||||
if head.info.name != bucket
|
||||
|| head.header.retired.is_some_and(|id| id.is_nil())
|
||||
|| head.header.manifest().is_some_and(|manifest| {
|
||||
manifest.bundle.is_nil() || manifest.length == 0 || manifest.level > 9 || head.header.retired == Some(manifest.bundle)
|
||||
})
|
||||
{
|
||||
return Err(invalid("invalid scanner checkpoint identity or size"));
|
||||
}
|
||||
Ok(head)
|
||||
}
|
||||
|
||||
async fn read_block<S: ScannerObjectIO>(store: Arc<S>, name: &str, bundle: uuid::Uuid, key: Digest) -> StorageResult<Bytes> {
|
||||
let path = block_path(name, bundle, &key);
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if reader.object_info.size < 1 || usize::try_from(reader.object_info.size).map_or(true, |size| size > PAGE_MAX) {
|
||||
return Err(invalid("invalid scanner checkpoint page length"));
|
||||
}
|
||||
let bytes = Bytes::from(reader.read_all().await?);
|
||||
if bytes.len() > PAGE_MAX || digest(&bytes) != key {
|
||||
return Err(invalid("scanner checkpoint page checksum mismatch"));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
async fn committed_keys<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
manifest: &Manifest,
|
||||
candidate: &Archive,
|
||||
) -> StorageResult<HashSet<Digest>> {
|
||||
let mut keys = vec![manifest.root];
|
||||
let mut known = HashSet::from([manifest.root]);
|
||||
for _ in 0..manifest.level {
|
||||
let parents = std::mem::take(&mut keys);
|
||||
for batch in parents.chunks(IO_CONCURRENCY) {
|
||||
let mut reads = Vec::new();
|
||||
let mut nodes = Vec::with_capacity(batch.len());
|
||||
for &key in batch {
|
||||
// The committed parent authenticates the table digest.
|
||||
// Unchanged tables are already available in the candidate.
|
||||
if let Some(node) = candidate.blocks.get(&key) {
|
||||
nodes.push(node.clone());
|
||||
} else {
|
||||
reads.push(read_block(store.clone(), name, manifest.bundle, key));
|
||||
}
|
||||
}
|
||||
nodes.extend(try_join_all(reads).await?);
|
||||
for node in nodes {
|
||||
if node.is_empty() || node.len() > FANOUT * 32 || node.len() % 32 != 0 {
|
||||
return Err(invalid("invalid scanner checkpoint page table"));
|
||||
}
|
||||
for &key in node.as_chunks::<32>().0 {
|
||||
keys.push(key);
|
||||
known.insert(key);
|
||||
}
|
||||
if u64::try_from(keys.len()).unwrap_or(u64::MAX) > manifest.length.div_ceil(PAGE_MIN as u64).saturating_add(3) {
|
||||
return Err(invalid("scanner checkpoint contains excess page references"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(known)
|
||||
}
|
||||
|
||||
pub(super) async fn decode<S: ScannerObjectIO>(store: Arc<S>, load_name: &str, bytes: &[u8]) -> StorageResult<DataUsageCache> {
|
||||
if !is_manifest(bytes) {
|
||||
return DataUsageCache::unmarshal(bytes).map_err(|_| invalid("invalid scanner cache"));
|
||||
}
|
||||
let name = load_name.strip_suffix(".bkp").unwrap_or(load_name);
|
||||
let bucket = bucket_cache_name(name)
|
||||
.and_then(|name| name.split_once('/'))
|
||||
.ok_or_else(|| invalid("misplaced scanner checkpoint"))?
|
||||
.0;
|
||||
let head = read_envelope(bytes, bucket)?;
|
||||
let Some(manifest) = head.header.manifest() else {
|
||||
return DataUsageCache::unmarshal(bytes).map_err(|_| invalid("invalid inline scanner checkpoint"));
|
||||
};
|
||||
let cache = restore::restore(store, load_name, manifest).await?;
|
||||
if cache.info.name != bucket
|
||||
|| cache.info.snapshot_complete != manifest.finished
|
||||
|| cache.info.next_cycle != manifest.cycle
|
||||
|| cache.info.scan_identity != head.info.scan_identity
|
||||
|| cache.info.source != head.info.source
|
||||
|| cache.info.leader_epoch != head.info.leader_epoch
|
||||
|| cache.info.next_cycle != head.info.next_cycle
|
||||
{
|
||||
return Err(invalid("scanner checkpoint scope changed"));
|
||||
}
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
async fn previous_header<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revision: &DataUsageCacheRevision,
|
||||
) -> StorageResult<Option<Header>> {
|
||||
if *revision == DataUsageCacheRevision::Missing {
|
||||
return Ok(None);
|
||||
}
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if reader
|
||||
.object_info
|
||||
.etag
|
||||
.as_ref()
|
||||
.map(|etag| DataUsageCacheRevision::Etag(etag.clone()))
|
||||
.as_ref()
|
||||
!= Some(revision)
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
// Header is the first value in the envelope. A large compatibility
|
||||
// projection or legacy cache does not require reading its body here.
|
||||
let mut prefix = Vec::new();
|
||||
(&mut reader).take(1024).read_to_end(&mut prefix).await?;
|
||||
if !is_manifest(&prefix) {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut deserializer = rmp_serde::Deserializer::new(&prefix[PREFIX.len()..]);
|
||||
// A damaged old header cannot participate in reuse or reclamation.
|
||||
// Rebuilding uses the CAS revision obtained by the normal loader.
|
||||
Ok(Header::deserialize(&mut deserializer).ok().filter(|header| {
|
||||
header.retired.is_none_or(|id| !id.is_nil())
|
||||
&& header
|
||||
.manifest()
|
||||
.is_none_or(|m| !m.bundle.is_nil() && m.length != 0 && m.level <= 9 && header.retired != Some(m.bundle))
|
||||
}))
|
||||
}
|
||||
|
||||
fn build_bundle(name: &str, revision: &DataUsageCacheRevision, cache: &DataUsageCache) -> uuid::Uuid {
|
||||
// Before the first root CAS, retries and competing builders populate the
|
||||
// same staging namespace. Individual page names are immutable digests.
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(b"rustfs-scanner-checkpoint-v1\0");
|
||||
hash.update(name.as_bytes());
|
||||
if let Some(identity) = cache.info.scan_identity {
|
||||
hash.update(identity.bucket_incarnation.as_bytes());
|
||||
}
|
||||
if let DataUsageCacheRevision::Etag(etag) = revision {
|
||||
hash.update(etag.as_bytes());
|
||||
}
|
||||
let mut id = [0; 16];
|
||||
id.copy_from_slice(&hash.finalize()[..16]);
|
||||
uuid::Uuid::from_bytes(id)
|
||||
}
|
||||
|
||||
pub(super) fn eligible(cache: &DataUsageCache, name: &str) -> bool {
|
||||
bucket_cache_name(name).is_some()
|
||||
&& cache.info.scan_identity.is_some()
|
||||
&& cache.info.name == name.split('/').next().unwrap_or("")
|
||||
}
|
||||
|
||||
pub(super) async fn prepare<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
cache: &DataUsageCache,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revision: &DataUsageCacheRevision,
|
||||
encoded: &EncodedPages,
|
||||
epoch: u64,
|
||||
damaged: bool,
|
||||
) -> StorageResult<(Bytes, Option<uuid::Uuid>)> {
|
||||
let previous_header = previous_header(store.clone(), name, revision).await?;
|
||||
let previous = previous_header.as_ref().and_then(Header::manifest);
|
||||
let mut retired = previous_header.as_ref().and_then(|header| header.retired);
|
||||
if let Some(id) = retired
|
||||
&& reclaim(store.clone(), name, id, epoch).await.is_ok()
|
||||
{
|
||||
retired = None;
|
||||
}
|
||||
// Each root has its own page namespace and can retire independently of
|
||||
// the other copy after a successful conditional commit.
|
||||
let allow_rotation = retired.is_none();
|
||||
if let Some(inline) = &encoded.inline
|
||||
&& (previous.is_none() || allow_rotation)
|
||||
{
|
||||
retired = retired.or_else(|| previous.map(|head| head.bundle));
|
||||
let bytes = if retired.is_some() {
|
||||
rmp_serde::to_vec_named(&CompatibilityEnvelope {
|
||||
header: &Header {
|
||||
body: Payload::Inline,
|
||||
retired,
|
||||
},
|
||||
info: &cache.info,
|
||||
cache: &cache.cache,
|
||||
})
|
||||
.map_err(|_| invalid("failed to encode inline scanner checkpoint"))?
|
||||
} else {
|
||||
return Ok((inline.clone(), None));
|
||||
};
|
||||
return Ok((Bytes::from(bytes), retired));
|
||||
}
|
||||
let live_bytes = save::block_bytes(encoded.archive.blocks.values());
|
||||
let reuse = match previous {
|
||||
Some(head) if !damaged && !(head.finished && head.cycle != cache.info.next_cycle) => {
|
||||
save::reuse(store.clone(), name, head, &encoded.archive, live_bytes).await?
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let verify_backup = name.ends_with(".bkp");
|
||||
let (bundle, known, bundle_bytes, bundle_commits) = if let Some((head, known, total)) = reuse {
|
||||
(head.bundle, known, total, head.bundle_commits + 1)
|
||||
} else {
|
||||
if let Some(head) = previous {
|
||||
if !allow_rotation {
|
||||
// Keep both readable generations and their cleanup obligation.
|
||||
// Further staging into either bundle would defeat the space bound.
|
||||
return Err(Error::other("cannot rotate scanner checkpoint before reclaiming its retired bundle"));
|
||||
}
|
||||
retired = Some(head.bundle);
|
||||
}
|
||||
(build_bundle(name, revision, cache), HashSet::new(), live_bytes, 1)
|
||||
};
|
||||
let mut manifest = encoded.root.clone();
|
||||
manifest.bundle = bundle;
|
||||
manifest.finished = cache.info.snapshot_complete;
|
||||
manifest.cycle = cache.info.next_cycle;
|
||||
manifest.bundle_bytes = bundle_bytes;
|
||||
manifest.bundle_commits = bundle_commits;
|
||||
|
||||
if let Err(error) = save::stage(store.clone(), name, revision, epoch, &manifest, &encoded.archive, &known).await {
|
||||
// A corrupt XL header need not yield an ETag for a conditional repair.
|
||||
// Rebuild this backup under a fresh namespace; its old root stays visible
|
||||
// until the manifest CAS. Other I/O failures remain retryable in place.
|
||||
if !(verify_backup && manifest.bundle_commits > 1 && allow_rotation && unreadable_page(&error)) {
|
||||
return Err(error);
|
||||
}
|
||||
retired = Some(manifest.bundle);
|
||||
manifest.bundle = build_bundle(name, revision, cache);
|
||||
manifest.bundle_bytes = live_bytes;
|
||||
manifest.bundle_commits = 1;
|
||||
save::stage(store.clone(), name, revision, epoch, &manifest, &encoded.archive, &HashSet::new()).await?;
|
||||
}
|
||||
let header = Header {
|
||||
body: Payload::Pages(manifest),
|
||||
retired,
|
||||
};
|
||||
let projection = |info: &DataUsageCacheInfo, cache: &BTreeMap<String, DataUsageEntry>| {
|
||||
rmp_serde::to_vec_named(&CompatibilityEnvelope {
|
||||
header: &header,
|
||||
info,
|
||||
cache,
|
||||
})
|
||||
.map_err(|_| invalid("failed to encode scanner checkpoint root"))
|
||||
};
|
||||
if cache.info.snapshot_complete {
|
||||
// Older readers can serve compact totals when the compatibility
|
||||
// projection fits. The manifest remains bounded even with many
|
||||
// pending repairs or a large lifecycle configuration.
|
||||
let mut root = cache
|
||||
.checked_flatten_complete_scope(&cache.info.name)
|
||||
.ok_or_else(|| invalid("disconnected complete bucket checkpoint"))?;
|
||||
root.compacted = true;
|
||||
let bytes = projection(&cache.info, &BTreeMap::from([(cache.info.name.clone(), root)]))?;
|
||||
if bytes.len() <= PAGE_MIN {
|
||||
return Ok((Bytes::from(bytes), retired));
|
||||
}
|
||||
}
|
||||
let info = DataUsageCacheInfo {
|
||||
name: cache.info.name.clone(),
|
||||
source: cache.info.source,
|
||||
scan_identity: cache.info.scan_identity,
|
||||
leader_epoch: cache.info.leader_epoch,
|
||||
next_cycle: cache.info.next_cycle,
|
||||
..Default::default()
|
||||
};
|
||||
let bytes = projection(&info, &BTreeMap::new())?;
|
||||
Ok((Bytes::from(bytes), retired))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CompatibilityEnvelope<'a, T> {
|
||||
#[serde(rename = "pagesV1")]
|
||||
header: &'a Header,
|
||||
info: &'a DataUsageCacheInfo,
|
||||
cache: &'a T,
|
||||
}
|
||||
|
||||
pub(super) async fn reclaim<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
bundle: uuid::Uuid,
|
||||
epoch: u64,
|
||||
) -> StorageResult<()> {
|
||||
let _admission = scanner_publication_admission_for_epoch(store.clone(), epoch)
|
||||
.await
|
||||
.ok_or_else(cache_publication_epoch_changed)?;
|
||||
let prefix = format!("{BUCKET_META_PREFIX}/{name}.pages/{bundle}/");
|
||||
let opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
};
|
||||
match timeout(
|
||||
DataUsageCache::cache_save_timeout(),
|
||||
store.delete_config_object(RUSTFS_META_BUCKET, &prefix, opts),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(_)) | Ok(Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..))) => Ok(()),
|
||||
Ok(Err(error)) => Err(error),
|
||||
Err(_) => Err(Error::other("retired scanner checkpoint cleanup timed out")),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::super::scan_load::park_restore;
|
||||
use super::*;
|
||||
use std::any::Any;
|
||||
use std::fs::File;
|
||||
use std::io::{Seek, SeekFrom};
|
||||
use std::sync::{Mutex, Weak};
|
||||
use tokio::time::Instant as WallInstant;
|
||||
|
||||
const MEMORY_LIMIT: u64 = 1024 * 1024;
|
||||
const PARKED_LIMIT: usize = 32;
|
||||
|
||||
enum Buffer {
|
||||
Memory(Vec<u8>),
|
||||
File(Arc<Mutex<File>>),
|
||||
}
|
||||
|
||||
struct Position {
|
||||
pending: Vec<(Digest, u8)>,
|
||||
length: u64,
|
||||
leaves: u64,
|
||||
nodes: u64,
|
||||
buffer: Buffer,
|
||||
writer: Option<tokio::task::JoinHandle<std::io::Result<()>>>,
|
||||
decoder: Option<tokio::task::JoinHandle<StorageResult<DataUsageCache>>>,
|
||||
}
|
||||
|
||||
struct Suspended {
|
||||
owner: Weak<dyn Any + Send + Sync>,
|
||||
name: String,
|
||||
manifest: Manifest,
|
||||
last_polled: WallInstant,
|
||||
position: Position,
|
||||
}
|
||||
|
||||
struct Recovery(Option<Suspended>);
|
||||
|
||||
static PARKED: LazyLock<Mutex<Vec<Suspended>>> = LazyLock::new(|| Mutex::new(Vec::new()));
|
||||
|
||||
impl Recovery {
|
||||
async fn open<S: ScannerObjectIO>(store: &Arc<S>, name: &str, manifest: &Manifest) -> StorageResult<Self> {
|
||||
let erased: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
let owner = Arc::downgrade(&erased);
|
||||
let resumed = {
|
||||
let mut parked = PARKED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
parked.retain(|job| {
|
||||
job.owner.strong_count() != 0 && (!job.owner.ptr_eq(&owner) || job.name != name || job.manifest == *manifest)
|
||||
});
|
||||
parked
|
||||
.iter()
|
||||
.position(|job| job.owner.ptr_eq(&owner) && job.name == name && job.manifest == *manifest)
|
||||
.map(|index| parked.swap_remove(index))
|
||||
};
|
||||
if let Some(mut job) = resumed {
|
||||
job.last_polled = WallInstant::now();
|
||||
return Ok(Self(Some(job)));
|
||||
}
|
||||
let buffer = if manifest.length <= MEMORY_LIMIT {
|
||||
Buffer::Memory(Vec::new())
|
||||
} else {
|
||||
let file = tokio::task::spawn_blocking(tempfile::tempfile)
|
||||
.await
|
||||
.map_err(Error::other)??;
|
||||
Buffer::File(Arc::new(Mutex::new(file)))
|
||||
};
|
||||
Ok(Self(Some(Suspended {
|
||||
owner,
|
||||
name: name.to_owned(),
|
||||
manifest: manifest.clone(),
|
||||
last_polled: WallInstant::now(),
|
||||
position: Position {
|
||||
pending: vec![(manifest.root, manifest.level)],
|
||||
length: 0,
|
||||
leaves: 0,
|
||||
nodes: 0,
|
||||
buffer,
|
||||
writer: None,
|
||||
decoder: None,
|
||||
},
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
// Park only verified bytes and traversal state. Outstanding metadata reads
|
||||
// and their admissions are dropped with the waiter. A local spool write or
|
||||
// decode may finish after cancellation and is joined before advancing again.
|
||||
impl Drop for Recovery {
|
||||
fn drop(&mut self) {
|
||||
let Some(mut job) = self.0.take() else { return };
|
||||
if job.position.nodes == 0 && job.position.leaves == 0 {
|
||||
return;
|
||||
}
|
||||
let mut parked = PARKED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
job.last_polled = WallInstant::now();
|
||||
if let Some(previous) = parked
|
||||
.iter_mut()
|
||||
.find(|previous| previous.owner.ptr_eq(&job.owner) && previous.name == job.name && previous.manifest == job.manifest)
|
||||
{
|
||||
// Readers of an immutable root traverse the same ordered prefix.
|
||||
// Keep the furthest admission regardless of cancellation order.
|
||||
if job.position.verified_progress() > previous.position.verified_progress() {
|
||||
std::mem::swap(previous, &mut job);
|
||||
}
|
||||
} else {
|
||||
let displaced = park_restore(&mut parked, PARKED_LIMIT, job, |job| job.last_polled);
|
||||
drop(parked);
|
||||
drop(displaced);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Position {
|
||||
fn verified_progress(&self) -> (u64, bool) {
|
||||
(self.nodes + self.leaves, self.decoder.is_some())
|
||||
}
|
||||
|
||||
async fn restore<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
manifest: &Manifest,
|
||||
) -> StorageResult<DataUsageCache> {
|
||||
let max_leaves = manifest.length.div_ceil(PAGE_MIN as u64).saturating_add(3);
|
||||
loop {
|
||||
// Retain the local write across cancellation. The verified page already
|
||||
// owns its offset and must not consume another storage read next round.
|
||||
if let Some(writer) = &mut self.writer {
|
||||
writer.await.map_err(Error::other)??;
|
||||
self.writer = None;
|
||||
}
|
||||
let Some(&(key, level)) = self.pending.last() else { break };
|
||||
if level != 0 {
|
||||
let node = read_block(store.clone(), name, manifest.bundle, key).await?;
|
||||
if node.is_empty()
|
||||
|| node.len() > FANOUT * 32
|
||||
|| node.len() % 32 != 0
|
||||
|| self.nodes >= max_leaves.saturating_mul(u64::from(manifest.level))
|
||||
{
|
||||
return Err(invalid("invalid scanner checkpoint restore table"));
|
||||
}
|
||||
self.pending.pop();
|
||||
for &child in node.as_chunks::<32>().0.iter().rev() {
|
||||
self.pending.push((child, level - 1));
|
||||
}
|
||||
self.nodes += 1;
|
||||
continue;
|
||||
}
|
||||
let block = read_block(store.clone(), name, manifest.bundle, key).await?;
|
||||
let next = self
|
||||
.length
|
||||
.checked_add(u64::try_from(block.len()).map_err(|_| invalid("invalid restore length"))?)
|
||||
.filter(|len| *len <= manifest.length)
|
||||
.ok_or_else(|| invalid("scanner checkpoint exceeds its committed length"))?;
|
||||
if self.leaves >= max_leaves {
|
||||
return Err(invalid("too many scanner checkpoint leaves"));
|
||||
}
|
||||
match &mut self.buffer {
|
||||
Buffer::Memory(bytes) => bytes.extend_from_slice(&block),
|
||||
Buffer::File(file) => {
|
||||
let file = file.clone();
|
||||
let offset = self.length;
|
||||
self.writer = Some(tokio::task::spawn_blocking(move || {
|
||||
let mut file = file.lock().unwrap_or_else(|error| error.into_inner());
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
file.write_all(&block)
|
||||
}));
|
||||
}
|
||||
}
|
||||
self.pending.pop();
|
||||
self.length = next;
|
||||
self.leaves += 1;
|
||||
}
|
||||
if self.length != manifest.length {
|
||||
return Err(invalid("incomplete scanner checkpoint"));
|
||||
}
|
||||
let job = self.decoder.get_or_insert_with(|| {
|
||||
let buffer = std::mem::replace(&mut self.buffer, Buffer::Memory(Vec::new()));
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = match buffer {
|
||||
Buffer::Memory(bytes) => rmp_serde::from_slice(&bytes),
|
||||
Buffer::File(file) => {
|
||||
let mut file = file.lock().unwrap_or_else(|error| error.into_inner());
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
rmp_serde::from_read(std::io::BufReader::new(&mut *file))
|
||||
}
|
||||
};
|
||||
result.map_err(|_| invalid("invalid scanner checkpoint payload"))
|
||||
})
|
||||
});
|
||||
job.await.map_err(Error::other)?
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn restore<S: ScannerObjectIO>(store: Arc<S>, name: &str, manifest: &Manifest) -> StorageResult<DataUsageCache> {
|
||||
let mut job = Recovery::open(&store, name, manifest).await?;
|
||||
let result = job
|
||||
.0
|
||||
.as_mut()
|
||||
.ok_or_else(|| invalid("missing checkpoint restore state"))?
|
||||
.position
|
||||
.restore(store, name, manifest)
|
||||
.await;
|
||||
job.0.take();
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
// A losing writer can stage blocks without publishing its accounting. Even
|
||||
// tiny or identical successful checkpoints periodically retire these orphans.
|
||||
const MAX_BUNDLE_COMMITS: u16 = 256;
|
||||
|
||||
pub(super) fn block_bytes<'a>(blocks: impl Iterator<Item = &'a Bytes>) -> u64 {
|
||||
blocks.fold(0u64, |sum, bytes| sum.saturating_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX)))
|
||||
}
|
||||
|
||||
pub(super) async fn reuse<'a, S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
head: &'a Manifest,
|
||||
candidate: &Archive,
|
||||
live: u64,
|
||||
) -> StorageResult<Option<(&'a Manifest, HashSet<Digest>, u64)>> {
|
||||
// Legacy manifests did not account for unreachable blocks in this bundle.
|
||||
if head.bundle_bytes == 0 || head.bundle_commits == 0 || head.bundle_commits >= MAX_BUNDLE_COMMITS {
|
||||
return Ok(None);
|
||||
}
|
||||
let known = match committed_keys(store, name, head, candidate).await {
|
||||
Ok(known) => known,
|
||||
Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..)) => return Ok(None),
|
||||
Err(error) if unreadable_page(&error) => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
// Count reachable allocation once at bundle creation and newly staged
|
||||
// bytes thereafter. A small delta stays incremental even across cycles.
|
||||
let added = block_bytes(
|
||||
candidate
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|(key, bytes)| (!known.contains(key)).then_some(bytes)),
|
||||
);
|
||||
let total = head.bundle_bytes.checked_add(added);
|
||||
Ok(total
|
||||
.filter(|&total| total <= live.saturating_mul(2))
|
||||
.map(|total| (head, known, total)))
|
||||
}
|
||||
|
||||
/// Returns the conditional write necessary to restore this content-addressed
|
||||
/// backup page, or None when the bytes in storage already match the candidate.
|
||||
async fn backup_revision<S: ScannerObjectIO>(
|
||||
store: &S,
|
||||
path: &str,
|
||||
expected: &Bytes,
|
||||
) -> StorageResult<Option<DataUsageCacheRevision>> {
|
||||
let mut reader = match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(reader) => reader,
|
||||
Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..)) => return Ok(Some(DataUsageCacheRevision::Missing)),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if usize::try_from(reader.object_info.size).ok() == Some(expected.len()) {
|
||||
let mut bytes = Vec::with_capacity(expected.len());
|
||||
(&mut reader).take(PAGE_MAX as u64 + 1).read_to_end(&mut bytes).await?;
|
||||
if bytes == expected.as_ref() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
let etag = reader
|
||||
.object_info
|
||||
.etag
|
||||
.ok_or_else(|| invalid("damaged scanner backup page has no revision"))?;
|
||||
Ok(Some(DataUsageCacheRevision::Etag(etag)))
|
||||
}
|
||||
|
||||
pub(super) async fn stage<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revision: &DataUsageCacheRevision,
|
||||
epoch: u64,
|
||||
manifest: &Manifest,
|
||||
archive: &Archive,
|
||||
known: &HashSet<Digest>,
|
||||
) -> StorageResult<()> {
|
||||
let verify_backup = name.ends_with(".bkp");
|
||||
let staging = staging::Staging::open(&store, name, manifest.bundle, epoch, revision);
|
||||
let mut writes = Vec::with_capacity(IO_CONCURRENCY);
|
||||
for (key, bytes) in &archive.blocks {
|
||||
let staged = staging.contains(key);
|
||||
if !verify_backup && (known.contains(key) || staged) {
|
||||
continue;
|
||||
}
|
||||
let path = block_path(name, manifest.bundle, key);
|
||||
let store = store.clone();
|
||||
let bytes = bytes.clone();
|
||||
let key = *key;
|
||||
let receipts = &staging;
|
||||
writes.push(async move {
|
||||
let needed = if verify_backup && (manifest.bundle_commits > 1 || staged) {
|
||||
// A healthy primary load does not establish the integrity of its
|
||||
// independent backup. Check reused bytes and repair only damaged pages.
|
||||
match timeout(DataUsageCache::cache_save_timeout(), backup_revision(store.as_ref(), &path, &bytes))
|
||||
.await
|
||||
.map_err(|_| Error::other("scanner backup page validation timed out"))??
|
||||
{
|
||||
Some(revision) => revision,
|
||||
None => return Ok(()),
|
||||
}
|
||||
} else {
|
||||
DataUsageCacheRevision::Missing
|
||||
};
|
||||
DataUsageCache::save_path_with_retry(
|
||||
store,
|
||||
&path,
|
||||
&bytes,
|
||||
DataUsageCache::cache_save_timeout(),
|
||||
DATA_USAGE_CACHE_SAVE_RETRIES,
|
||||
Some(needed),
|
||||
Some(epoch),
|
||||
)
|
||||
.await?;
|
||||
receipts.record(key);
|
||||
Ok::<(), Error>(())
|
||||
});
|
||||
if writes.len() == IO_CONCURRENCY {
|
||||
try_join_all(std::mem::take(&mut writes)).await?;
|
||||
}
|
||||
}
|
||||
try_join_all(writes).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use std::any::Any;
|
||||
use std::sync::{Mutex, Weak};
|
||||
use std::time::Instant as WallInstant;
|
||||
|
||||
/// A manifest commit can be interrupted after any group of immutable PUTs.
|
||||
/// Reuse only receipts from the same store, epoch and predecessor revision.
|
||||
struct Attempt {
|
||||
store: Weak<dyn Any + Send + Sync>,
|
||||
path: String,
|
||||
bundle: uuid::Uuid,
|
||||
epoch: u64,
|
||||
revision: DataUsageCacheRevision,
|
||||
seen: Arc<Mutex<HashSet<Digest>>>,
|
||||
last_used: WallInstant,
|
||||
}
|
||||
|
||||
const MAX_ATTEMPTS: usize = 32;
|
||||
static ATTEMPTS: LazyLock<Mutex<Vec<Attempt>>> = LazyLock::new(|| Mutex::new(Vec::new()));
|
||||
|
||||
pub(super) struct Staging(Arc<Mutex<HashSet<Digest>>>);
|
||||
|
||||
impl Staging {
|
||||
pub(super) fn open<S: ScannerObjectIO>(
|
||||
store: &Arc<S>,
|
||||
path: &str,
|
||||
bundle: uuid::Uuid,
|
||||
epoch: u64,
|
||||
revision: &DataUsageCacheRevision,
|
||||
) -> Self {
|
||||
let store: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
let owner = Arc::downgrade(&store);
|
||||
let mut attempts = ATTEMPTS.lock().unwrap_or_else(|p| p.into_inner());
|
||||
attempts.retain(|job| job.store.strong_count() != 0 && job.last_used.elapsed() < Duration::from_secs(3600));
|
||||
if let Some(attempt) = attempts.iter_mut().find(|attempt| {
|
||||
attempt.store.ptr_eq(&owner)
|
||||
&& attempt.path == path
|
||||
&& attempt.bundle == bundle
|
||||
&& attempt.epoch == epoch
|
||||
&& &attempt.revision == revision
|
||||
}) {
|
||||
attempt.last_used = WallInstant::now();
|
||||
return Self(attempt.seen.clone());
|
||||
}
|
||||
// A successfully committed predecessor naturally retires the receipts
|
||||
// for its previous ETag. Concurrent older writers retain their own Arc.
|
||||
attempts.retain(|attempt| !attempt.store.ptr_eq(&owner) || attempt.path != path);
|
||||
if attempts.len() >= MAX_ATTEMPTS
|
||||
&& let Some((index, _)) = attempts.iter().enumerate().min_by_key(|(_, job)| job.last_used)
|
||||
{
|
||||
attempts.swap_remove(index);
|
||||
}
|
||||
let seen = Arc::new(Mutex::new(HashSet::new()));
|
||||
attempts.push(Attempt {
|
||||
store: owner,
|
||||
path: path.to_owned(),
|
||||
bundle,
|
||||
epoch,
|
||||
revision: revision.clone(),
|
||||
seen: seen.clone(),
|
||||
last_used: WallInstant::now(),
|
||||
});
|
||||
Self(seen)
|
||||
}
|
||||
|
||||
pub(super) fn contains(&self, key: &Digest) -> bool {
|
||||
self.0.lock().unwrap_or_else(|p| p.into_inner()).contains(key)
|
||||
}
|
||||
|
||||
pub(super) fn record(&self, key: Digest) {
|
||||
self.0.lock().unwrap_or_else(|p| p.into_inner()).insert(key);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudget;
|
||||
use std::any::Any;
|
||||
use std::sync::{Mutex, Weak};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
// Keep only a failed root's identity while its independent backup is being
|
||||
// recovered. A fresh primary reader still obtains the current ETag every time.
|
||||
pub(super) struct PendingBackup {
|
||||
owner: Weak<dyn Any + Send + Sync>,
|
||||
name: String,
|
||||
failed_etag: String,
|
||||
last_polled: tokio::time::Instant,
|
||||
}
|
||||
|
||||
static PENDING_BACKUPS: Mutex<Vec<PendingBackup>> = Mutex::new(Vec::new());
|
||||
const MAX_PENDING_BACKUPS: usize = 32;
|
||||
|
||||
// Age permits reclamation under contention. A scanner whose configured
|
||||
// interval exceeds this age can still resume while slots remain available.
|
||||
pub(super) fn park_restore<T>(
|
||||
slots: &mut Vec<T>,
|
||||
limit: usize,
|
||||
value: T,
|
||||
last_polled: impl Fn(&T) -> tokio::time::Instant,
|
||||
) -> Option<T> {
|
||||
if slots.len() < limit {
|
||||
slots.push(value);
|
||||
return None;
|
||||
}
|
||||
if let Some(oldest) = slots.iter_mut().min_by_key(|slot| last_polled(slot))
|
||||
&& last_polled(oldest).elapsed() >= Duration::from_secs(3600)
|
||||
{
|
||||
return Some(std::mem::replace(oldest, value));
|
||||
}
|
||||
Some(value)
|
||||
}
|
||||
|
||||
impl PendingBackup {
|
||||
fn owner<S: ScannerObjectIO>(store: &Arc<S>) -> Weak<dyn Any + Send + Sync> {
|
||||
let erased: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
Arc::downgrade(&erased)
|
||||
}
|
||||
|
||||
pub(super) fn needs_backup<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) -> bool {
|
||||
let owner = Self::owner(store);
|
||||
let mut pending = PENDING_BACKUPS.lock().unwrap_or_else(|error| error.into_inner());
|
||||
let Some(index) = pending
|
||||
.iter()
|
||||
.position(|phase| phase.owner.ptr_eq(&owner) && phase.name == name)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if pending[index].failed_etag == etag {
|
||||
pending[index].last_polled = tokio::time::Instant::now();
|
||||
true
|
||||
} else {
|
||||
pending.swap_remove(index);
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn begin<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) {
|
||||
let owner = Self::owner(store);
|
||||
let mut pending = PENDING_BACKUPS.lock().unwrap_or_else(|error| error.into_inner());
|
||||
pending.retain(|phase| phase.owner.strong_count() > 0 && !(phase.owner.ptr_eq(&owner) && phase.name == name));
|
||||
let phase = Self {
|
||||
owner,
|
||||
name: name.to_owned(),
|
||||
failed_etag: etag.to_owned(),
|
||||
last_polled: tokio::time::Instant::now(),
|
||||
};
|
||||
let _ = park_restore(&mut pending, MAX_PENDING_BACKUPS, phase, |phase| phase.last_polled);
|
||||
}
|
||||
|
||||
pub(super) fn finish<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) {
|
||||
let owner = Self::owner(store);
|
||||
PENDING_BACKUPS
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.retain(|phase| !(phase.owner.ptr_eq(&owner) && phase.name == name && phase.failed_etag == etag));
|
||||
}
|
||||
}
|
||||
|
||||
struct DecodedCheckpoint {
|
||||
owner: Weak<dyn Any + Send + Sync>,
|
||||
name: String,
|
||||
parked_at: std::time::Instant,
|
||||
cache: DataUsageCache,
|
||||
revisions: DataUsageCacheRevisions,
|
||||
}
|
||||
|
||||
// The scanner may finish a cold restore with too little time for the walk.
|
||||
// Move that exact value to the next admission. Only one decoded bucket can
|
||||
// occupy the handoff and it never owns storage, a permit, or a publication fence.
|
||||
static DECODED: Mutex<Option<DecodedCheckpoint>> = Mutex::new(None);
|
||||
|
||||
impl DataUsageCache {
|
||||
pub(crate) async fn load_for_scan<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
ctx: &CancellationToken,
|
||||
budget: &ScannerCycleBudget,
|
||||
) -> StorageResult<Option<DataUsageCacheRevisions>> {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = ctx.cancelled() => Ok(None),
|
||||
result = self.load_for_scan_window(store, name, budget) => result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_for_scan_window<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
budget: &ScannerCycleBudget,
|
||||
) -> StorageResult<Option<DataUsageCacheRevisions>> {
|
||||
let erased: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
let owner = Arc::downgrade(&erased);
|
||||
let is_ours = |ready: &DecodedCheckpoint| ready.owner.ptr_eq(&owner) && ready.name == name;
|
||||
let cached = {
|
||||
let mut decoded = DECODED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
// Give a retired or deleted bucket's slot back to other active
|
||||
// readers. A matching admission can consume it even after a long
|
||||
// scheduled interval, subject to the revision checks below.
|
||||
decoded.take_if(|ready| {
|
||||
ready.owner.strong_count() == 0 || (!is_ours(ready) && ready.parked_at.elapsed() >= Duration::from_secs(3600))
|
||||
});
|
||||
decoded.as_ref().is_some_and(is_ours)
|
||||
};
|
||||
if cached {
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let main = read_config_revision(store.clone(), &path).await;
|
||||
let backup = Self::backup_revision(store.clone(), name).await;
|
||||
let ready = DECODED
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.take_if(|ready| is_ours(ready));
|
||||
if let (Some(ready), Ok(main)) = (ready, main)
|
||||
&& ready.revisions.main == main
|
||||
&& ready.revisions.backup == backup
|
||||
{
|
||||
*self = ready.cache;
|
||||
return Ok(Some(ready.revisions));
|
||||
}
|
||||
}
|
||||
|
||||
let started = tokio::time::Instant::now();
|
||||
// Obtain the optional backup head before the expensive decode. A budget
|
||||
// stop after the decoder finishes must not discard the materialized tree
|
||||
// while waiting for an unrelated backup lookup.
|
||||
let backup = Self::backup_revision(store.clone(), name).await;
|
||||
let loaded = Box::pin(Self::load_cache(store, name)).await?;
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: loaded.main_revision,
|
||||
backup: loaded.backup_revision.or(backup),
|
||||
damaged_pages: loaded.damaged_pages,
|
||||
};
|
||||
*self = loaded.cache;
|
||||
if self.info.scan_identity.is_some()
|
||||
&& !self.cache.is_empty()
|
||||
&& budget
|
||||
.remaining_config()
|
||||
.max_duration
|
||||
.is_some_and(|remaining| started.elapsed() > remaining)
|
||||
{
|
||||
let mut slot = DECODED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if slot.as_ref().is_none_or(|ready| ready.owner.strong_count() == 0) {
|
||||
*slot = Some(DecodedCheckpoint {
|
||||
owner,
|
||||
name: name.to_owned(),
|
||||
parked_at: std::time::Instant::now(),
|
||||
cache: std::mem::take(self),
|
||||
revisions,
|
||||
});
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Ok(Some(revisions))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn only_reclaim_idle_restore_slots_under_pressure() {
|
||||
let first = tokio::time::Instant::now();
|
||||
tokio::time::advance(Duration::from_secs(7200)).await;
|
||||
let second = tokio::time::Instant::now();
|
||||
let mut slots = vec![first];
|
||||
assert_eq!(park_restore(&mut slots, 2, second, |&time| time), None);
|
||||
assert_eq!(slots, [first, second]);
|
||||
let new = second + Duration::from_secs(1);
|
||||
assert_eq!(park_restore(&mut slots, 2, new, |&time| time), Some(first));
|
||||
assert_eq!(slots, [new, second]);
|
||||
assert_eq!(park_restore(&mut slots, 2, new, |&time| time), Some(new));
|
||||
assert_eq!(slots, [new, second], "active recoveries retain both slots");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A linked tree paused on a different physical replica. It contains only
|
||||
/// traversal fields, so dormant sources cannot recursively contain each other.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct SourceTree {
|
||||
bucket: String,
|
||||
identity: DataUsageScanIdentity,
|
||||
started_plan: DataUsageScanPlanDigest,
|
||||
#[serde(serialize_with = "ordered_entries")]
|
||||
entries: HashMap<String, DataUsageEntry>,
|
||||
resume_after: Option<String>,
|
||||
checkpoint: Option<DataUsageScanCheckpoint>,
|
||||
coverage: Option<DataUsageScanCoverageReceipt>,
|
||||
cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
index: Option<RawEnumerationPageIndex>,
|
||||
ancestors: Vec<RawEnumerationPageIndex>,
|
||||
}
|
||||
|
||||
impl SourceTree {
|
||||
/// An inactive receipt may preserve an existing source position. Only an
|
||||
/// active, verified traversal can contribute new progress.
|
||||
pub(super) fn frontier(&self, info: &DataUsageCacheInfo) -> Option<&str> {
|
||||
if self.bucket != info.name
|
||||
|| Some(self.identity) != info.scan_identity
|
||||
|| Some(self.started_plan) != info.scan_progress.map(|p| p.started_plan)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
self.coverage.as_ref().map(|receipt| receipt.through.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
// Dormant trees live in the checkpoint header stream. A cold process has
|
||||
// fresh HashMap seeds; preserve stable bytes for immutable Merkle pages.
|
||||
fn ordered_entries<S: serde::Serializer>(
|
||||
entries: &HashMap<String, DataUsageEntry>,
|
||||
serializer: S,
|
||||
) -> std::result::Result<S::Ok, S::Error> {
|
||||
let mut sorted = entries.iter().collect::<Vec<_>>();
|
||||
sorted.sort_unstable_by(|a, b| a.0.cmp(b.0));
|
||||
serializer.collect_map(sorted.into_iter().map(|(name, entry)| (name, entry.ordered_encoding())))
|
||||
}
|
||||
|
||||
impl DataUsageCache {
|
||||
pub(crate) fn activate_directory_source(&mut self, source: String, endpoints: &HashSet<String>) {
|
||||
let (Some(identity), Some(progress)) = (self.info.scan_identity, self.info.scan_progress) else {
|
||||
return;
|
||||
};
|
||||
let previous = self.info.scan_directory_source.replace(source.clone());
|
||||
// Every production endpoint has an erasure-set slot. A disk replacement
|
||||
// cannot retain a backup under the same slot or under a retired endpoint.
|
||||
let current_location = directory_source_location(&source);
|
||||
self.info.scan_directory_work.retain(|work| {
|
||||
work.endpoint() == source
|
||||
|| endpoints.contains(directory_source_location(work.endpoint()))
|
||||
&& directory_source_location(work.endpoint()) != current_location
|
||||
});
|
||||
self.info.scan_source_trees.retain(|key, tree| {
|
||||
tree.bucket == self.info.name
|
||||
&& tree.identity == identity
|
||||
&& tree.started_plan == progress.started_plan
|
||||
&& (key == &source
|
||||
|| endpoints.contains(directory_source_location(key)) && directory_source_location(key) != current_location)
|
||||
});
|
||||
let Some(previous) = previous else {
|
||||
// Pre-source checkpoints identify an erasure set but never recorded
|
||||
// which disk supplied the partial tree. The same keys can have
|
||||
// different metadata on another replica. Keep the certified usage,
|
||||
// repair obligations and observations outside this traversal.
|
||||
if self.has_scan_position() {
|
||||
self.clear_active_directory_traversal();
|
||||
}
|
||||
return;
|
||||
};
|
||||
if previous == source {
|
||||
return;
|
||||
}
|
||||
let entries = std::mem::take(&mut self.cache);
|
||||
let linked = entries.get(&hash_path(&self.info.name).0).is_some_and(|root| !root.compacted);
|
||||
let previous_tree = SourceTree {
|
||||
bucket: self.info.name.clone(),
|
||||
identity,
|
||||
started_plan: progress.started_plan,
|
||||
entries,
|
||||
resume_after: self.info.scan_resume_after.take(),
|
||||
checkpoint: self.info.scan_checkpoint.take(),
|
||||
coverage: self.info.scan_coverage_receipt.take(),
|
||||
cursor: self.info.scan_raw_enumeration_cursor.take(),
|
||||
index: self.info.scan_raw_enumeration_page_index.take(),
|
||||
ancestors: std::mem::take(&mut self.info.scan_raw_enumeration_path_page_indexes),
|
||||
};
|
||||
if linked
|
||||
&& endpoints.contains(directory_source_location(&previous))
|
||||
&& directory_source_location(&previous) != current_location
|
||||
{
|
||||
self.info.scan_source_trees.insert(previous, Arc::new(previous_tree));
|
||||
}
|
||||
if let Some(tree) = self.info.scan_source_trees.remove(&source) {
|
||||
let tree = Arc::unwrap_or_clone(tree);
|
||||
self.cache = tree.entries;
|
||||
self.info.scan_resume_after = tree.resume_after;
|
||||
self.info.scan_coverage_receipt = tree.coverage;
|
||||
self.info.scan_checkpoint = tree.checkpoint;
|
||||
self.info.scan_raw_enumeration_cursor = tree.cursor;
|
||||
self.info.scan_raw_enumeration_page_index = tree.index;
|
||||
self.info.scan_raw_enumeration_path_page_indexes = tree.ancestors;
|
||||
// A dormant tree must provide the same coverage/graph guarantees
|
||||
// required of an active checkpoint before its child links are used.
|
||||
if self.checked_flatten_complete_scope(&self.info.name).is_none()
|
||||
|| self.info.scan_coverage_receipt.is_some() && self.validated_scan_frontier().is_none()
|
||||
{
|
||||
self.clear_active_directory_traversal();
|
||||
}
|
||||
}
|
||||
}
|
||||
fn clear_active_directory_traversal(&mut self) {
|
||||
self.cache.clear();
|
||||
self.info.scan_resume_after = None;
|
||||
self.info.scan_checkpoint = None;
|
||||
self.info.scan_coverage_receipt = None;
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_source_location(source: &str) -> &str {
|
||||
source.split_once('|').map_or(source, |(_, endpoint)| endpoint)
|
||||
}
|
||||
@@ -653,6 +653,7 @@ async fn cache_save_reconciles_an_ambiguous_committed_write() {
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: None,
|
||||
};
|
||||
|
||||
@@ -772,102 +773,6 @@ async fn corrupt_primary_cache_loads_valid_backup() {
|
||||
assert_eq!(store.puts.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn baseline_revisions_allow_only_a_decoded_unchanged_primary_backup_catchup() {
|
||||
let etag = |value: &str| DataUsageCacheRevision::Etag(value.to_owned());
|
||||
let initial = DataUsageCacheBaselineRevisions {
|
||||
revisions: DataUsageCacheRevisions {
|
||||
main: etag("primary"),
|
||||
backup: Some(etag("old-backup")),
|
||||
},
|
||||
loaded_from_main: true,
|
||||
};
|
||||
assert!(initial.permits_unchanged_baseline_since(&initial));
|
||||
let mut caught_up = initial.clone();
|
||||
caught_up.revisions.backup = Some(etag("primary"));
|
||||
assert!(caught_up.permits_unchanged_baseline_since(&initial));
|
||||
assert!(
|
||||
!initial.permits_unchanged_baseline_since(&caught_up),
|
||||
"backup rollback changes the baseline"
|
||||
);
|
||||
for backup in [None, Some(DataUsageCacheRevision::Missing)] {
|
||||
let mut missing_backup = initial.clone();
|
||||
missing_backup.revisions.backup = backup;
|
||||
assert!(caught_up.permits_unchanged_baseline_since(&missing_backup));
|
||||
assert!(!missing_backup.permits_unchanged_baseline_since(&initial));
|
||||
}
|
||||
for (initial_from_main, current_from_main) in [(false, true), (true, false), (false, false)] {
|
||||
let mut baseline = initial.clone();
|
||||
baseline.loaded_from_main = initial_from_main;
|
||||
let mut current = caught_up.clone();
|
||||
current.loaded_from_main = current_from_main;
|
||||
assert!(
|
||||
!current.permits_unchanged_baseline_since(&baseline),
|
||||
"backup fallback grants no primary proof"
|
||||
);
|
||||
current.revisions = baseline.revisions.clone();
|
||||
assert!(
|
||||
current.permits_unchanged_baseline_since(&baseline),
|
||||
"exact revisions retain existing admission"
|
||||
);
|
||||
}
|
||||
let mut changed_main = caught_up.clone();
|
||||
changed_main.revisions.main = etag("new-primary");
|
||||
changed_main.revisions.backup = Some(etag("new-primary"));
|
||||
assert!(!changed_main.permits_unchanged_baseline_since(&initial));
|
||||
let mut changed_backup = caught_up;
|
||||
changed_backup.revisions.backup = Some(etag("unrelated-backup"));
|
||||
assert!(!changed_backup.permits_unchanged_baseline_since(&initial));
|
||||
for main in [DataUsageCacheRevision::Missing, etag("")] {
|
||||
let mut baseline = initial.clone();
|
||||
baseline.revisions.main = main.clone();
|
||||
let mut current = baseline.clone();
|
||||
current.revisions.backup = Some(main);
|
||||
assert!(
|
||||
!current.permits_unchanged_baseline_since(&baseline),
|
||||
"missing or empty primary ETag is not proof"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn baseline_revisions_record_the_actual_decoded_cache_source() {
|
||||
let mut expected = DataUsageCache::default();
|
||||
expected.info.name = "decoded".to_owned();
|
||||
let valid = expected.marshal_msg().expect("encode valid cache");
|
||||
for (main, backup, from_main, name) in [
|
||||
(valid.clone(), Some(valid.clone()), true, "decoded"),
|
||||
(b"corrupt".to_vec(), Some(valid.clone()), false, "decoded"),
|
||||
(b"corrupt".to_vec(), None, false, ""),
|
||||
(Vec::new(), None, false, ""),
|
||||
] {
|
||||
let store = Arc::new(CacheReadStore::new(CacheReadBody::Bytes(main), backup));
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let baseline = loaded
|
||||
.load_with_baseline_revisions(store.clone(), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("load actual cache source");
|
||||
assert_eq!(baseline.loaded_from_main, from_main);
|
||||
assert_eq!(loaded.info.name, name);
|
||||
let mut raw_loaded = DataUsageCache::default();
|
||||
let raw = raw_loaded
|
||||
.load_with_revisions(store, DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("load compatible raw revisions");
|
||||
assert_eq!(baseline.revisions(), &raw, "raw loader retains its revision contract");
|
||||
}
|
||||
let mut legacy_store = CacheReadStore::new(CacheReadBody::Bytes(Vec::new()), None);
|
||||
legacy_store.legacy = Some(valid);
|
||||
let mut legacy_cache = DataUsageCache::default();
|
||||
let legacy = legacy_cache
|
||||
.load_with_baseline_revisions(Arc::new(legacy_store), "legacy-cache")
|
||||
.await
|
||||
.expect("load cache from the legacy location");
|
||||
assert_eq!(legacy_cache.info.name, "decoded");
|
||||
assert!(!legacy.loaded_from_main);
|
||||
assert_eq!(legacy.revisions.main, DataUsageCacheRevision::Missing);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_cache_without_valid_backup_rebuilds_with_cas_revisions() {
|
||||
for backup in [None, Some(b"also-not-msgpack".to_vec())] {
|
||||
@@ -2976,6 +2881,7 @@ async fn test_save_backup_budget_tracks_primary_cost() {
|
||||
let store = Arc::new(store);
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: Some(DataUsageCacheRevision::Etag("backup-etag".to_string())),
|
||||
};
|
||||
DataUsageCache::default()
|
||||
@@ -3028,6 +2934,7 @@ async fn test_checkpoint_budget_covers_retries_and_reconciliation() {
|
||||
let store = Arc::new(store);
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: Some(DataUsageCacheRevision::Etag("backup-etag".to_string())),
|
||||
};
|
||||
let started = Instant::now();
|
||||
|
||||
@@ -88,6 +88,7 @@ async fn check_save(store: BackupFallbackStore, revisions: Option<DataUsageCache
|
||||
fn revisions(backup: bool) -> DataUsageCacheRevisions {
|
||||
DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: backup.then_some(DataUsageCacheRevision::Missing),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::data_usage_define::clone_cache_value;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::sync::Arc;
|
||||
use thiserror::Error;
|
||||
|
||||
mod writer;
|
||||
@@ -58,7 +59,7 @@ pub struct RawEnumerationPage {
|
||||
parent: String,
|
||||
page_index: u64,
|
||||
entries_start: u64,
|
||||
entries: Vec<String>,
|
||||
entries: Arc<[String]>,
|
||||
terminal: bool,
|
||||
digest: [u8; 32],
|
||||
}
|
||||
@@ -518,32 +519,26 @@ impl RawEnumerationPage {
|
||||
}
|
||||
|
||||
fn new(parent: &str, building: RawEnumerationPageBuilder) -> Self {
|
||||
let digest = raw_page_digest(parent, &building);
|
||||
let digest = raw_page_digest(parent, building.page_index, building.entries_start, building.terminal, &building.entries);
|
||||
Self {
|
||||
version: RAW_PAGE_INDEX_VERSION,
|
||||
parent: clone_cache_value(parent),
|
||||
page_index: building.page_index,
|
||||
entries_start: building.entries_start,
|
||||
entries: building.entries,
|
||||
entries: building.entries.into(),
|
||||
terminal: building.terminal,
|
||||
digest,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(&self, parent: &str, page_index: u64, entries_start: u64) -> Result<(), RawEnumerationPageIndexError> {
|
||||
let builder = RawEnumerationPageBuilder {
|
||||
page_index: self.page_index,
|
||||
entries_start: self.entries_start,
|
||||
entries: clone_cache_value(&self.entries),
|
||||
terminal: self.terminal,
|
||||
};
|
||||
if self.version != RAW_PAGE_INDEX_VERSION
|
||||
|| self.parent != parent
|
||||
|| self.page_index != page_index
|
||||
|| self.entries_start != entries_start
|
||||
|| self.entries.is_empty()
|
||||
|| !entries_are_normalized(&self.entries)
|
||||
|| self.digest != raw_page_digest(parent, &builder)
|
||||
|| self.digest != raw_page_digest(parent, self.page_index, self.entries_start, self.terminal, &self.entries)
|
||||
{
|
||||
return Err(RawEnumerationPageIndexError::CorruptIndex);
|
||||
}
|
||||
@@ -587,16 +582,16 @@ fn entry_sets_match(left: &[String], right: &[String]) -> bool {
|
||||
left.len() == right.len() && entries_contain_all(left, right)
|
||||
}
|
||||
|
||||
fn raw_page_digest(parent: &str, building: &RawEnumerationPageBuilder) -> [u8; 32] {
|
||||
fn raw_page_digest(parent: &str, page_index: u64, entries_start: u64, terminal: bool, entries: &[String]) -> [u8; 32] {
|
||||
#[cfg(test)]
|
||||
RAW_PAGE_DIGEST_ENTRIES.with(|count| count.set(count.get().saturating_add(building.entries.len())));
|
||||
RAW_PAGE_DIGEST_ENTRIES.with(|count| count.set(count.get().saturating_add(entries.len())));
|
||||
let mut digest = Sha256::new();
|
||||
update_digest(&mut digest, b"version", &RAW_PAGE_INDEX_VERSION.to_le_bytes());
|
||||
update_digest(&mut digest, b"parent", parent.as_bytes());
|
||||
update_digest(&mut digest, b"page_index", &building.page_index.to_le_bytes());
|
||||
update_digest(&mut digest, b"entries_start", &building.entries_start.to_le_bytes());
|
||||
update_digest(&mut digest, b"terminal", &[u8::from(building.terminal)]);
|
||||
for entry in &building.entries {
|
||||
update_digest(&mut digest, b"page_index", &page_index.to_le_bytes());
|
||||
update_digest(&mut digest, b"entries_start", &entries_start.to_le_bytes());
|
||||
update_digest(&mut digest, b"terminal", &[u8::from(terminal)]);
|
||||
for entry in entries {
|
||||
update_digest(&mut digest, b"entry", entry.as_bytes());
|
||||
}
|
||||
digest.finalize().into()
|
||||
|
||||
@@ -64,6 +64,15 @@ impl RawEnumerationPageWriter {
|
||||
})
|
||||
}
|
||||
|
||||
/// A parked native stream delivers the unread tail first. Incomplete
|
||||
/// membership can immediately append that tail; terminal page verification
|
||||
/// still requires observing the entire current directory.
|
||||
pub(crate) fn continue_native_stream(&mut self) {
|
||||
if !self.inner.complete && !self.inner.building.as_ref().is_some_and(|page| page.terminal) {
|
||||
self.observations = self.revalidate_after_entries;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn indexed_entry_count(&self) -> usize {
|
||||
self.indexed.len()
|
||||
}
|
||||
|
||||
@@ -956,9 +956,14 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
}
|
||||
})?;
|
||||
let mut cache = DataUsageCache::default();
|
||||
let mut revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
|
||||
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
|
||||
})?;
|
||||
let Some(mut revisions) = Box::pin(cache.load_for_scan(set.clone(), &cache_name, &ctx, &budget))
|
||||
.await
|
||||
.map_err(|err| {
|
||||
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
|
||||
})?
|
||||
else {
|
||||
return Ok(RemoteScannerFrameResult::Partial);
|
||||
};
|
||||
let expected_incarnation = cache
|
||||
.info
|
||||
.scan_identity
|
||||
|
||||
+124
-40
@@ -34,8 +34,9 @@ use crate::runtime_config::{
|
||||
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
|
||||
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
|
||||
use crate::scanner_io::{
|
||||
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, dirty_usage_bucket_notified, dirty_usage_buckets_pending,
|
||||
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
|
||||
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, ScannerServiceCohort, dirty_usage_bucket_notified,
|
||||
dirty_usage_buckets_pending, dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
|
||||
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
|
||||
@@ -55,7 +56,7 @@ use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
CurrentCycle, Metric, Metrics, RemoteScanProgress, ScanCyclePartialReason, ScanCycleWorkSnapshot,
|
||||
ScannerUsagePublicationObservation, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete,
|
||||
emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
|
||||
emit_scan_cycle_deferred, emit_scan_cycle_partial_with_stage, emit_scan_cycle_superseded, global_metrics,
|
||||
};
|
||||
use rustfs_scanner_metrics::startup::ScannerStartupToken;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -248,13 +249,13 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
|
||||
/// otherwise idle instance whose clean-idle backoff had doubled a 60 s
|
||||
/// interval), which defeats the fast path it is meant to protect.
|
||||
///
|
||||
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] protects
|
||||
/// against a persistently hot bucket driving an unbroken full-scan loop, so
|
||||
/// retries can start small: 5 s, 10 s, 20 s … Superseded retries cap this
|
||||
/// schedule at the ordinary cycle interval and preserve an operator-selected
|
||||
/// nominal cadence. Preflight deferrals retain the generic
|
||||
/// [`SCANNER_RETRY_MAX_INTERVAL`] cap so a temporarily unavailable peer cannot
|
||||
/// drive a tight retry loop.
|
||||
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] is
|
||||
/// what protects against a persistently hot bucket driving an unbroken
|
||||
/// full-scan loop, so it can start small: 5 s, 10 s, 20 s … capped by
|
||||
/// [`SCANNER_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a
|
||||
/// genuinely hot bucket still reaches minute-scale backoff within a handful of
|
||||
/// cycles. Preflight deferrals use the same bounded schedule so a temporarily
|
||||
/// unavailable peer cannot drive a tight retry loop.
|
||||
const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
/// Probe cadence while data movement pauses the scanner. Recovery retries use
|
||||
@@ -1212,6 +1213,8 @@ where
|
||||
}
|
||||
|
||||
let mut transient_backoff = ScannerRetryBackoff::default();
|
||||
// Only this supervisor, for this storage instance, owns resumable service.
|
||||
let mut service_continuation = None;
|
||||
let mut recovery_retry_count = 0_u32;
|
||||
loop {
|
||||
if ctx_clone.is_cancelled() {
|
||||
@@ -1226,6 +1229,7 @@ where
|
||||
startup_features,
|
||||
startup_maintenance_generation,
|
||||
startup,
|
||||
&mut service_continuation,
|
||||
)
|
||||
.await;
|
||||
if let Err(e) = &run_result {
|
||||
@@ -1773,11 +1777,26 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Scheduling and mutation witnesses retained across a clean local deadline.
|
||||
/// Only an immediately succeeding leadership claim can resume this cohort.
|
||||
struct ScannerServiceContinuation {
|
||||
cohort: Arc<StdMutex<ScannerServiceCohort>>,
|
||||
fenced_epoch: u64,
|
||||
fenced_revision: DataUsageCacheRevision,
|
||||
}
|
||||
|
||||
impl ScannerServiceContinuation {
|
||||
fn matches_claim(&self, previous_revision: &DataUsageCacheRevision, claimed_epoch: u64) -> bool {
|
||||
&self.fenced_revision == previous_revision && self.fenced_epoch.checked_add(1) == Some(claimed_epoch)
|
||||
}
|
||||
}
|
||||
|
||||
struct ScannerCycleDeadlineState<'a> {
|
||||
cycle_info: &'a mut CurrentCycle,
|
||||
cycle_revision: &'a mut DataUsageCacheRevision,
|
||||
leader_epoch: &'a mut u64,
|
||||
cycle_budget: &'a ScannerCycleBudget,
|
||||
service_cohort: Arc<StdMutex<ScannerServiceCohort>>,
|
||||
allow_bootstrap_pending: bool,
|
||||
}
|
||||
|
||||
@@ -1791,7 +1810,8 @@ async fn handle_scanner_cycle_deadline<Store>(
|
||||
state: ScannerCycleDeadlineState<'_>,
|
||||
worker_stopped: bool,
|
||||
guard: &mut NamespaceLockGuard,
|
||||
) where
|
||||
) -> Option<ScannerServiceContinuation>
|
||||
where
|
||||
Store: ScannerObjectIO + ScannerConfigObjectDelete,
|
||||
{
|
||||
let fenced = fence_scanner_epoch_after_cycle_timeout(
|
||||
@@ -1821,8 +1841,14 @@ async fn handle_scanner_cycle_deadline<Store>(
|
||||
global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
|
||||
// Stop renewing before releasing the lease. A new leader can then claim the
|
||||
// higher persisted generation instead of inheriting the expired worker.
|
||||
let may_resume = !recovery_required && !guard.is_lock_lost() && !ctx.is_cancelled();
|
||||
guard.release();
|
||||
global_metrics().set_cycle(None).await;
|
||||
may_resume.then(|| ScannerServiceContinuation {
|
||||
cohort: state.service_cohort,
|
||||
fenced_epoch: *state.leader_epoch,
|
||||
fenced_revision: state.cycle_revision.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
|
||||
@@ -2460,13 +2486,20 @@ where
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
required_cycle,
|
||||
state = "cache_cycle_ahead",
|
||||
partial_stage = "cache_cycle_ahead",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
required_cycle,
|
||||
partial_cause = "cache_cycle_ahead",
|
||||
"Scanner cycle is recovering to a newer durable cache generation"
|
||||
);
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let partial = emit_scan_cycle_partial_with_stage(
|
||||
cycle_start.elapsed(),
|
||||
ScanCyclePartialReason::Unknown,
|
||||
None,
|
||||
Some("cache_cycle_ahead"),
|
||||
);
|
||||
let persisted = persist_required_scanner_cycle_floor_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
@@ -2480,6 +2513,7 @@ where
|
||||
},
|
||||
)
|
||||
.await;
|
||||
partial.observe_state_write(persisted);
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
@@ -2530,18 +2564,7 @@ where
|
||||
}
|
||||
|
||||
usage_publication_result.restrict_outcome(usage_persist_outcome);
|
||||
let partial_cause = if scan_cycle_result.has_partial_bucket_progress() && !scan_cycle_result.has_usage_update() {
|
||||
"saved_bucket_progress"
|
||||
} else if scan_cycle_result.has_usage_update()
|
||||
&& matches!(
|
||||
scan_cycle_result.status,
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
)
|
||||
{
|
||||
"activity_unverified_observation"
|
||||
} else {
|
||||
"incomplete_coverage"
|
||||
};
|
||||
let (partial_cause, partial_stage) = partial_cycle_diagnostic(&scan_cycle_result);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let durable_observation = scan_cycle_result
|
||||
.publication_expectation()
|
||||
@@ -2620,25 +2643,35 @@ where
|
||||
);
|
||||
}
|
||||
if cycle_budget.budget_elapsed() {
|
||||
let budget_reason = cycle_budget.reason();
|
||||
let budget_stage = match budget_reason {
|
||||
Some(ScannerCycleBudgetReason::Runtime) => Some("runtime_budget"),
|
||||
Some(ScannerCycleBudgetReason::Objects) => Some("object_budget"),
|
||||
Some(ScannerCycleBudgetReason::Directories) => Some("directory_budget"),
|
||||
Some(ScannerCycleBudgetReason::Entries) => Some("entry_budget"),
|
||||
None => None,
|
||||
};
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "budget_reached",
|
||||
partial_stage = budget_stage,
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
duration = ?now.elapsed(),
|
||||
reason = ?cycle_budget.reason(),
|
||||
reason = ?budget_reason,
|
||||
max_duration = ?cycle_budget.max_duration(),
|
||||
max_objects = ?cycle_budget.max_objects(),
|
||||
max_directories = ?cycle_budget.max_directories(),
|
||||
state = "budget_reached",
|
||||
"Scanner cycle budget reached"
|
||||
);
|
||||
let budget_reason = cycle_budget.reason();
|
||||
emit_scan_cycle_partial_with_source(
|
||||
let partial = emit_scan_cycle_partial_with_stage(
|
||||
cycle_start.elapsed(),
|
||||
scan_cycle_partial_reason(budget_reason),
|
||||
scan_cycle_partial_source(budget_reason),
|
||||
budget_stage,
|
||||
);
|
||||
let persisted = finalize_partial_scan_cycle_for_epoch(
|
||||
ctx,
|
||||
@@ -2650,6 +2683,7 @@ where
|
||||
publication_epoch,
|
||||
)
|
||||
.await;
|
||||
partial.observe_state_write(persisted);
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
@@ -2683,29 +2717,56 @@ where
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
ScannerCycleOutcome::Partial => {
|
||||
if ctx.is_cancelled() {
|
||||
let owner_cancelled = ctx.is_cancelled();
|
||||
let partial_stage = if owner_cancelled { "owner_cancelled" } else { partial_stage };
|
||||
if owner_cancelled {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "incomplete_cancelled",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
partial_stage,
|
||||
"Scanner cycle stopped before a complete usage snapshot was produced"
|
||||
);
|
||||
} else {
|
||||
} else if partial_cause == "saved_bucket_progress"
|
||||
&& matches!(partial_stage, "bucket_checkpoint_saved" | "bucket_walk_incomplete")
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "incomplete",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
partial_cause,
|
||||
"Scanner cycle ended without a complete usage snapshot"
|
||||
partial_stage,
|
||||
"Scanner saved bucket progress before completing coverage"
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "incomplete",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
partial_cause,
|
||||
partial_stage,
|
||||
persist_outcome = ?usage_persist_outcome,
|
||||
"Scanner cycle could not certify complete usage"
|
||||
);
|
||||
}
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let partial = emit_scan_cycle_partial_with_stage(
|
||||
cycle_start.elapsed(),
|
||||
ScanCyclePartialReason::Unknown,
|
||||
None,
|
||||
Some(partial_stage),
|
||||
);
|
||||
let persisted = finalize_partial_scan_cycle_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
@@ -2716,6 +2777,7 @@ where
|
||||
publication_epoch,
|
||||
)
|
||||
.await;
|
||||
partial.observe_state_write(persisted);
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
@@ -2909,6 +2971,7 @@ where
|
||||
maintenance_features,
|
||||
maintenance_generation,
|
||||
runtime_guard.startup_token(),
|
||||
&mut None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2995,10 +3058,12 @@ async fn run_data_scanner_with_maintenance_state<S>(
|
||||
mut maintenance_features: ScannerMaintenanceFeatures,
|
||||
mut maintenance_generation_seen: Option<u64>,
|
||||
startup: ScannerStartupToken,
|
||||
continuation: &mut Option<ScannerServiceContinuation>,
|
||||
) -> Result<(), ScannerError>
|
||||
where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
let previous = continuation.take();
|
||||
reset_scanner_cycle_schedule();
|
||||
// Normal scanner entry is an attempt, not evidence of leadership, coverage, or publication.
|
||||
global_metrics().startup().record_attempt(startup);
|
||||
@@ -3108,7 +3173,6 @@ where
|
||||
let mut deferred_backoff = ScannerRetryBackoff::default();
|
||||
let mut scope_fallback_backoff = ScannerRetryBackoff::default();
|
||||
let scope_fallback_slot = Arc::new(ScannerScopeFallbackSlot::default());
|
||||
let service_cohort = Arc::new(StdMutex::new(crate::scanner_io::ScannerServiceCohort::default()));
|
||||
let initial_runtime_config = resolve_scanner_runtime_config();
|
||||
if clean_idle_topology_supported && maintenance_generation_seen.is_none() {
|
||||
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
|
||||
@@ -3269,6 +3333,7 @@ where
|
||||
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
||||
return Ok(());
|
||||
}
|
||||
let previous_revision = cycle_revision.clone();
|
||||
let claim_ctx = ctx.child_token();
|
||||
let leadership_claimed = await_scanner_cycle_with_lock_fence(
|
||||
&claim_ctx,
|
||||
@@ -3318,6 +3383,10 @@ where
|
||||
finish_scanner_leader_iteration(false, "epoch_claim_failed", "leadership epoch claim failed".to_string()).await;
|
||||
return Ok(());
|
||||
}
|
||||
let service_cohort = previous
|
||||
.filter(|previous| previous.matches_claim(&previous_revision, leader_epoch))
|
||||
.map(|previous| previous.cohort)
|
||||
.unwrap_or_default();
|
||||
if rebuild_from_quarantine {
|
||||
record_scanner_cycle_state_rebuilt_from_quarantine(cycle_info.next, leader_epoch);
|
||||
}
|
||||
@@ -3402,7 +3471,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
*continuation = handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
@@ -3410,6 +3479,7 @@ where
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
service_cohort,
|
||||
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
||||
},
|
||||
worker_stopped,
|
||||
@@ -3528,7 +3598,8 @@ where
|
||||
));
|
||||
let mut wait_plan =
|
||||
scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for);
|
||||
let superseded_retry_interval = scanner_superseded_retry_interval(superseded_backoff, &runtime_config);
|
||||
let superseded_retry_interval =
|
||||
scanner_superseded_retry_interval(superseded_backoff, &runtime_config, maintenance_features);
|
||||
let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval);
|
||||
let scope_fallback_retry_interval = scanner_scope_fallback_retry_interval(scope_fallback_backoff, &runtime_config);
|
||||
let mut convergence_retry_interval = superseded_retry_interval
|
||||
@@ -3746,7 +3817,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
*continuation = handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
@@ -3754,6 +3825,7 @@ where
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
service_cohort,
|
||||
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
||||
},
|
||||
worker_stopped,
|
||||
@@ -4333,6 +4405,18 @@ fn finalize_scanner_cycle_result(
|
||||
)
|
||||
}
|
||||
|
||||
fn partial_cycle_diagnostic(scan: &crate::scanner_io::ScannerCycleResult) -> (&'static str, &'static str) {
|
||||
if scan.has_partial_bucket_progress() && !scan.has_usage_update() {
|
||||
("saved_bucket_progress", scan.partial_stage.unwrap_or("bucket_checkpoint_saved"))
|
||||
} else if scan.has_usage_update()
|
||||
&& scan.status == ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
{
|
||||
("activity_unverified_observation", "activity_baseline_unverified")
|
||||
} else {
|
||||
("incomplete_coverage", scan.partial_stage.unwrap_or("root_coverage_incomplete"))
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_cycle_completion_outcome_for_result(
|
||||
scan_cycle_result: &crate::scanner_io::ScannerCycleResult,
|
||||
usage_persist_outcome: DataUsagePersistOutcome,
|
||||
|
||||
@@ -192,16 +192,19 @@ impl ScannerRetryBackoff {
|
||||
pub(super) fn scanner_superseded_retry_interval(
|
||||
backoff: ScannerRetryBackoff,
|
||||
runtime_config: &ScannerRuntimeConfig,
|
||||
maintenance_features: ScannerMaintenanceFeatures,
|
||||
) -> Option<Duration> {
|
||||
let cycle_interval = runtime_config.cycle_interval.max(Duration::from_secs(1));
|
||||
let retry_interval = backoff.retry_interval(cycle_interval)?;
|
||||
// Preserve an operator-selected nominal cadence during convergence.
|
||||
let mut retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?;
|
||||
// Preserve an operator-selected duty-cycle floor during convergence.
|
||||
if runtime_config.cycle_interval_source != ScannerRuntimeConfigSource::Default {
|
||||
return Some(cycle_interval);
|
||||
retry_interval = retry_interval.max(runtime_config.cycle_interval);
|
||||
}
|
||||
// A complete but non-authoritative walk must not postpone convergence
|
||||
// beyond the ordinary cadence, even without lifecycle or replication.
|
||||
Some(retry_interval.min(cycle_interval))
|
||||
// A complete but non-authoritative walk must not postpone lifecycle or
|
||||
// replication evaluation beyond its ordinary maintenance cadence.
|
||||
if maintenance_features.needs_regular_cycle() {
|
||||
retry_interval = retry_interval.min(runtime_config.cycle_interval.max(Duration::from_secs(1)));
|
||||
}
|
||||
Some(retry_interval)
|
||||
}
|
||||
|
||||
/// Pacing for the cycle after a full-namespace scope fallback while dirty
|
||||
|
||||
@@ -52,9 +52,11 @@ const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
|
||||
mod cycle_budget_publication;
|
||||
pub(super) mod cycle_persist_failure;
|
||||
mod gate_self_heal;
|
||||
mod paged_checkpoint;
|
||||
mod quota_reset_preservation;
|
||||
|
||||
mod recovery_control;
|
||||
mod restore_deadline;
|
||||
mod scoped_ack_publication;
|
||||
mod wakeup_metrics;
|
||||
|
||||
@@ -773,7 +775,8 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
|
||||
);
|
||||
budget.mark_cycle_state_persisted();
|
||||
|
||||
handle_scanner_cycle_deadline(
|
||||
let service_cohort = Arc::new(StdMutex::new(ScannerServiceCohort::default()));
|
||||
let continuation = handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
store.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
@@ -781,12 +784,24 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &budget,
|
||||
service_cohort: service_cohort.clone(),
|
||||
allow_bootstrap_pending: false,
|
||||
},
|
||||
true,
|
||||
&mut guard,
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.expect("a fully drained and fenced deadline retains the live cohort across supervisor iterations");
|
||||
assert!(Arc::ptr_eq(&continuation.cohort, &service_cohort));
|
||||
assert!(continuation.matches_claim(&cycle_revision, 3));
|
||||
assert!(
|
||||
!continuation.matches_claim(&cycle_revision, 4),
|
||||
"an intervening owner ends the continuation"
|
||||
);
|
||||
assert!(
|
||||
!continuation.matches_claim(&DataUsageCacheRevision::Missing, 3),
|
||||
"a replaced cursor ends the continuation"
|
||||
);
|
||||
|
||||
assert!(guard.is_released());
|
||||
assert_eq!(leader_epoch, 2, "deadline handler should claim the next epoch");
|
||||
@@ -989,6 +1004,7 @@ struct MemoryConfigStore {
|
||||
insert_after_gets: Mutex<HashMap<String, Vec<u8>>>,
|
||||
read_errors: Mutex<HashMap<String, EcstoreError>>,
|
||||
delayed_gets: Mutex<HashMap<String, Duration>>,
|
||||
checkpoint_page_delay_ms: AtomicUsize,
|
||||
non_regular_objects: Mutex<HashSet<String>>,
|
||||
fail_put_number: Mutex<HashMap<String, usize>>,
|
||||
/// Fail every put whose number is at least the stored value.
|
||||
@@ -1000,6 +1016,7 @@ struct MemoryConfigStore {
|
||||
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
|
||||
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
||||
error_after_commit_deletes: Mutex<HashSet<String>>,
|
||||
fail_prefix_deletes: AtomicBool,
|
||||
cancel_after_deletes: Mutex<HashMap<String, CancellationToken>>,
|
||||
pause_next_publication_admission: Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>,
|
||||
put_counts: Mutex<HashMap<String, usize>>,
|
||||
@@ -1046,6 +1063,17 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
|
||||
.await
|
||||
.push(crate::storage_api::ecstore_diagnostic::current_read_version_entry_source());
|
||||
let key = memory_config_key(bucket, object);
|
||||
let _serial_page_read = if object.contains(".pages/") {
|
||||
restore_deadline::admit_page(object).await
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if object.contains(".pages/") {
|
||||
let delay = self.checkpoint_page_delay_ms.load(Ordering::Relaxed);
|
||||
if delay != 0 {
|
||||
tokio::time::sleep(Duration::from_millis(delay as u64)).await;
|
||||
}
|
||||
}
|
||||
if let Some(error) = self.read_errors.lock().await.get(&key).cloned() {
|
||||
return Err(error);
|
||||
}
|
||||
@@ -5003,6 +5031,15 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
||||
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
|
||||
let key = memory_config_key(bucket, object);
|
||||
let mut objects = self.objects.lock().await;
|
||||
if opts.delete_prefix && !opts.delete_prefix_object {
|
||||
if self.fail_prefix_deletes.load(Ordering::Relaxed) {
|
||||
return Err(EcstoreError::other("injected prefix deletion failure"));
|
||||
}
|
||||
assert!(opts.http_preconditions.is_none(), "fixture prefix deletion cannot condition one object");
|
||||
objects.retain(|object, _| !object.starts_with(&key));
|
||||
self.revisions.lock().await.retain(|object, _| !object.starts_with(&key));
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if !objects.contains_key(&key) {
|
||||
return Err(EcstoreError::FileNotFound);
|
||||
}
|
||||
@@ -8947,7 +8984,7 @@ fn clean_idle_backoff_grows_to_cap() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_backoff_grows_caps_and_resets_after_convergence() {
|
||||
fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() {
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
|
||||
|
||||
@@ -8971,7 +9008,7 @@ fn retry_backoff_grows_caps_and_resets_after_convergence() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_backoff_respects_a_faster_configured_cycle() {
|
||||
fn superseded_retry_backoff_respects_a_faster_configured_cycle() {
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
backoff.record_retryable_cycle(true);
|
||||
|
||||
@@ -8983,7 +9020,7 @@ fn retry_backoff_respects_a_faster_configured_cycle() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retry_backoff_grows_from_the_default_cycle() {
|
||||
fn superseded_retry_backoff_grows_from_the_default_cycle() {
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
// The first race after a write burst retries in seconds, not a whole
|
||||
// cycle, while repeated supersedes still climb toward the cap.
|
||||
@@ -9003,7 +9040,7 @@ fn superseded_retry_preserves_explicit_cycle_cadence() {
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
scanner_superseded_retry_interval(backoff, &default_config),
|
||||
scanner_superseded_retry_interval(backoff, &default_config, ScannerMaintenanceFeatures::default()),
|
||||
Some(SCANNER_RETRY_BASE_INTERVAL),
|
||||
"the default adaptive cadence must retain fast convergence"
|
||||
);
|
||||
@@ -9013,24 +9050,24 @@ fn superseded_retry_preserves_explicit_cycle_cadence() {
|
||||
ScannerRuntimeConfigSource::Config,
|
||||
ScannerRuntimeConfigSource::ScannerCompatConfig,
|
||||
] {
|
||||
for cycle_secs in [1, 3, 600, 6 * 60 * 60] {
|
||||
let runtime_config = ScannerRuntimeConfig {
|
||||
cycle_interval: Duration::from_secs(cycle_secs),
|
||||
cycle_interval_source: source,
|
||||
let runtime_config = ScannerRuntimeConfig {
|
||||
cycle_interval: Duration::from_secs(6 * 60 * 60),
|
||||
cycle_interval_source: source,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for features in [
|
||||
ScannerMaintenanceFeatures::default(),
|
||||
ScannerMaintenanceFeatures {
|
||||
lifecycle: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
assert_eq!(scanner_superseded_retry_interval(backoff, &runtime_config), None);
|
||||
for _ in 0..40 {
|
||||
backoff.record_retryable_cycle(true);
|
||||
assert_eq!(
|
||||
scanner_superseded_retry_interval(backoff, &runtime_config),
|
||||
Some(runtime_config.cycle_interval),
|
||||
"{source:?} cycle cadence must hold throughout convergence retries"
|
||||
);
|
||||
}
|
||||
backoff.record_retryable_cycle(false);
|
||||
assert_eq!(scanner_superseded_retry_interval(backoff, &runtime_config), None);
|
||||
},
|
||||
] {
|
||||
assert_eq!(
|
||||
scanner_superseded_retry_interval(backoff, &runtime_config, features),
|
||||
Some(runtime_config.cycle_interval),
|
||||
"{source:?} cycle cadence must not be shortened by convergence retries"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9051,7 +9088,7 @@ fn superseded_retry_does_not_delay_required_maintenance() {
|
||||
for consecutive_cycles in 1..=12 {
|
||||
backoff.record_retryable_cycle(true);
|
||||
assert_eq!(
|
||||
scanner_superseded_retry_interval(backoff, &config),
|
||||
scanner_superseded_retry_interval(backoff, &config, features),
|
||||
Some(config.cycle_interval),
|
||||
"superseded cycle {consecutive_cycles} must preserve regular maintenance"
|
||||
);
|
||||
@@ -9137,50 +9174,19 @@ fn scope_fallback_retry_preserves_explicit_and_fast_cycle_cadence() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn superseded_retry_caps_at_the_default_cycle_and_resets_after_convergence() {
|
||||
fn superseded_retry_only_caps_regular_maintenance() {
|
||||
let config = ScannerRuntimeConfig {
|
||||
cycle_interval: Duration::from_secs(60),
|
||||
..Default::default()
|
||||
};
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
assert_eq!(scanner_superseded_retry_interval(backoff, &config), None);
|
||||
for expected in [5, 10, 20, 40, 60, 60, 60, 60] {
|
||||
backoff.record_retryable_cycle(true);
|
||||
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(Duration::from_secs(expected)));
|
||||
}
|
||||
backoff.consecutive_cycles = u32::MAX;
|
||||
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(config.cycle_interval));
|
||||
backoff.record_retryable_cycle(true);
|
||||
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(config.cycle_interval));
|
||||
backoff.record_retryable_cycle(false);
|
||||
assert_eq!(scanner_superseded_retry_interval(backoff, &config), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn superseded_retry_caps_fast_cycles_without_changing_the_generic_retry_cap() {
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
for _ in 0..40 {
|
||||
for _ in 0..6 {
|
||||
backoff.record_retryable_cycle(true);
|
||||
}
|
||||
assert_eq!(
|
||||
backoff.retry_interval(Duration::from_secs(60)),
|
||||
Some(Duration::from_secs(30 * 60)),
|
||||
"deferred and transient retries must retain the generic cap"
|
||||
scanner_superseded_retry_interval(backoff, &config, ScannerMaintenanceFeatures::default()),
|
||||
Some(Duration::from_secs(160)),
|
||||
);
|
||||
for cycle_secs in [0, 1, 3, 60, 6 * 60 * 60] {
|
||||
let config = ScannerRuntimeConfig {
|
||||
cycle_interval: Duration::from_secs(cycle_secs),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
scanner_superseded_retry_interval(backoff, &config),
|
||||
Some(Duration::from_secs(cycle_secs.max(1))),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn superseded_retry_timer_keeps_required_maintenance_full_scans() {
|
||||
for features in [
|
||||
ScannerMaintenanceFeatures {
|
||||
lifecycle: true,
|
||||
@@ -9195,23 +9201,28 @@ fn superseded_retry_timer_keeps_required_maintenance_full_scans() {
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert!(features.requires_full_scan(Some(7), 7, ScannerCycleWakeReason::Timer, true));
|
||||
assert_eq!(scanner_superseded_retry_interval(backoff, &config, features), Some(config.cycle_interval));
|
||||
assert_eq!(scanner_superseded_retry_interval(ScannerRetryBackoff::default(), &config, features), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn superseded_retry_wait_keeps_convergence_fencing_at_the_default_cycle() {
|
||||
async fn superseded_maintenance_wait_keeps_convergence_fencing() {
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
let config = ScannerRuntimeConfig {
|
||||
cycle_interval: Duration::from_secs(60),
|
||||
..Default::default()
|
||||
};
|
||||
let features = ScannerMaintenanceFeatures {
|
||||
lifecycle: true,
|
||||
..Default::default()
|
||||
};
|
||||
let mut backoff = ScannerRetryBackoff::default();
|
||||
for _ in 0..12 {
|
||||
backoff.record_retryable_cycle(true);
|
||||
}
|
||||
let interval = scanner_superseded_retry_interval(backoff, &config);
|
||||
let interval = scanner_superseded_retry_interval(backoff, &config, features);
|
||||
assert_eq!(interval, Some(config.cycle_interval));
|
||||
let generations = ScannerCycleObservedGenerations::for_wait(
|
||||
&config,
|
||||
@@ -9226,28 +9237,19 @@ async fn superseded_retry_wait_keeps_convergence_fencing_at_the_default_cycle()
|
||||
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
|
||||
let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]);
|
||||
let started = tokio::time::Instant::now();
|
||||
let mut wait = Box::pin(wait_for_next_scanner_cycle_with_activity(
|
||||
let reason = wait_for_next_scanner_cycle_with_activity(
|
||||
&ctx,
|
||||
interval.expect("superseded retry must retain its convergence interval"),
|
||||
interval.expect("maintenance must retain its convergence interval"),
|
||||
Some(Duration::from_secs(1)),
|
||||
&mut seen,
|
||||
generations,
|
||||
|| false,
|
||||
|| std::future::ready(Ok(changed.clone())),
|
||||
));
|
||||
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
|
||||
for _ in 0..3 {
|
||||
tokio::time::advance(Duration::from_secs(15)).await;
|
||||
crate::scanner_io::record_dirty_usage_bucket("photos");
|
||||
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
|
||||
}
|
||||
let reason = wait.await;
|
||||
)
|
||||
.await;
|
||||
assert_eq!(reason, ScannerCycleWakeReason::Timer);
|
||||
assert_eq!(started.elapsed(), config.cycle_interval);
|
||||
assert_eq!(seen, Some(changed));
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -253,6 +253,9 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
.await;
|
||||
assert!(matches!(outcome, ScannerCycleOutcome::Partial));
|
||||
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
|
||||
let report = global_metrics().report().await;
|
||||
assert_eq!(report.last_cycle_partial_reason, "objects");
|
||||
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("object_budget"));
|
||||
assert!(budget.progress().0 > 0);
|
||||
assert!(budget.cycle_state_persisted());
|
||||
assert_eq!(cycle.next, 353);
|
||||
@@ -410,6 +413,16 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
super::cycle_persist_failure::fail_saves(1, false, SCANNER_PERSIST_CAS_RETRIES + 1)
|
||||
}
|
||||
});
|
||||
use tracing::instrument::WithSubscriber as _;
|
||||
let error_log = tempfile::NamedTempFile::new().expect("error log");
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.json()
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_max_level(tracing::Level::ERROR)
|
||||
.with_writer(error_log.reopen().expect("error log writer"))
|
||||
.finish();
|
||||
let _interest_guard = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());
|
||||
let tail_outcome = run_data_scanner_cycle_with_budget(
|
||||
&ctx,
|
||||
&store,
|
||||
@@ -423,7 +436,23 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
scope_fallback: None,
|
||||
},
|
||||
)
|
||||
.with_subscriber(subscriber)
|
||||
.await;
|
||||
let error_events = std::fs::read_to_string(error_log.path()).expect("read error log");
|
||||
let event = error_events
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON error event"))
|
||||
.find(|event| event["fields"]["state"] == "incomplete" && event["fields"]["cycle"] == 353)
|
||||
.expect("unbudgeted incomplete observation should be visible at error level");
|
||||
assert_eq!(event["fields"]["leader_epoch"], 1);
|
||||
assert_eq!(event["fields"]["partial_stage"], "mixed_plan_recertification_pending");
|
||||
let report = global_metrics().report().await;
|
||||
assert_eq!(report.last_cycle_partial_reason, "unknown");
|
||||
if fail_cycle_save == Some(false) {
|
||||
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("cycle_state_not_durable"));
|
||||
} else {
|
||||
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("mixed_plan_recertification_pending"));
|
||||
}
|
||||
assert!(!tail_budget.budget_elapsed(), "this is not the budget-exhaustion fast path");
|
||||
checkpoint.load(set, &cache_name).await.expect("reload the saved mixed tail");
|
||||
assert!(!checkpoint.info.snapshot_complete);
|
||||
@@ -527,6 +556,22 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_diagnostics_keep_the_activity_fence_distinct_from_checkpoint_progress() {
|
||||
let observation =
|
||||
ScannerCycleResult::new(ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable), None)
|
||||
.with_usage_update_emitted(true);
|
||||
assert_eq!(
|
||||
partial_cycle_diagnostic(&observation),
|
||||
("activity_unverified_observation", "activity_baseline_unverified")
|
||||
);
|
||||
let checkpoint = ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_partial_bucket_progress(true);
|
||||
assert_eq!(
|
||||
partial_cycle_diagnostic(&checkpoint),
|
||||
("saved_bucket_progress", "bucket_checkpoint_saved")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_progress_does_not_mask_missing_updates_or_persistence_failure() {
|
||||
for (persisted, expected) in [
|
||||
|
||||
@@ -0,0 +1,473 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::BUCKET_META_PREFIX;
|
||||
|
||||
mod save_review;
|
||||
|
||||
const NAME: &str = "bucket/.usage-cache.bin";
|
||||
|
||||
fn empty_bucket_checkpoint() -> DataUsageCache {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "bucket".into();
|
||||
cache.info.source = Some(DataUsageCacheSource::new(0, 0));
|
||||
cache.info.scan_identity = Some(crate::data_usage_define::DataUsageScanIdentity {
|
||||
version: crate::data_usage_define::DATA_USAGE_SCAN_CHECKPOINT_VERSION,
|
||||
bucket_incarnation: uuid::Uuid::from_u128(42),
|
||||
set_layout: DataUsageScanPlanDigest([7; 32]),
|
||||
publication_epoch: 0,
|
||||
tier_registry_generation: 1,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
});
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
cache
|
||||
}
|
||||
|
||||
fn seeded() -> DataUsageCache {
|
||||
let mut cache = empty_bucket_checkpoint();
|
||||
for index in 0..4096 {
|
||||
cache.replace(
|
||||
&format!("bucket/prefix-{index:05}"),
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: index + 1,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
cache
|
||||
}
|
||||
|
||||
async fn layout(store: &MemoryConfigStore) -> HashMap<String, (usize, usize)> {
|
||||
let puts = store.put_counts.lock().await.clone();
|
||||
store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), (puts[k], v.len())))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn page_bytes_after_save(before: &HashMap<String, (usize, usize)>, after: &HashMap<String, (usize, usize)>) -> (usize, usize) {
|
||||
let mut existing = 0;
|
||||
let mut uploaded = 0;
|
||||
for (path, &(writes, size)) in after.iter().filter(|(path, _)| path.contains(".pages/")) {
|
||||
if let Some(&(previous_writes, _)) = before.get(path) {
|
||||
assert_eq!(writes, previous_writes, "a committed immutable page was resubmitted");
|
||||
existing += size;
|
||||
} else {
|
||||
uploaded += size;
|
||||
}
|
||||
}
|
||||
(existing, uploaded)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compacted_checkpoint_advances_one_child_without_rewriting_300k_names() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = empty_bucket_checkpoint();
|
||||
let plan = DataUsageScanPlanDigest([1; 32]);
|
||||
cache.info.scan_progress = Some(crate::DataUsageScanProgress {
|
||||
started_plan: plan,
|
||||
requested_plan: plan,
|
||||
});
|
||||
cache.info.scan_directory_source = Some(format!("{}|disk-a", uuid::Uuid::from_u128(1)));
|
||||
let [first, second] = crate::scanner_folder::DirectoryCheckpoint::wide_storage_fixture(&cache.info).await;
|
||||
let mut before: Option<HashMap<String, (usize, usize)>> = None;
|
||||
let mut prior_progress = None;
|
||||
for (objects, receipt) in [(121_034, first), (121_035, second)] {
|
||||
cache.info.scan_directory_work = vec![receipt];
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
compacted: true,
|
||||
objects,
|
||||
size: objects * 1024,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("current revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save compacted receipt");
|
||||
let after = layout(&store).await;
|
||||
if let Some(before) = before.as_ref() {
|
||||
let (old_bytes, new_bytes) = page_bytes_after_save(before, &after);
|
||||
eprintln!("300k-name checkpoint: {new_bytes} bytes in new pages, {old_bytes} bytes in existing pages");
|
||||
assert!(old_bytes > 3_000_000, "fixture must contain the full physical listing");
|
||||
assert!(
|
||||
new_bytes > 0 && new_bytes < old_bytes / 10,
|
||||
"one more completed child wrote {new_bytes} bytes alongside {old_bytes} reused bytes"
|
||||
);
|
||||
}
|
||||
before = Some(after);
|
||||
let mut restored = DataUsageCache::default();
|
||||
restored
|
||||
.load(store.clone(), NAME)
|
||||
.await
|
||||
.expect("cold read of compacted receipt");
|
||||
assert!(restored.info.scan_directory_work[0].is_bound_to(&restored.info));
|
||||
let progress = restored.durable_scan_progress();
|
||||
if let Some(prior) = &prior_progress {
|
||||
assert!(progress.advanced_from(prior), "the new child must earn another durable work turn");
|
||||
}
|
||||
prior_progress = Some(progress);
|
||||
let total = restored.checked_flatten("bucket").expect("connected root after restore");
|
||||
assert_eq!((total.objects, total.size), (objects, objects * 1024));
|
||||
cache = restored;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dormant_linked_tree_pages_survive_source_switch_and_cold_reencoding() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
let plan = DataUsageScanPlanDigest([1; 32]);
|
||||
cache.info.scan_progress = Some(crate::DataUsageScanProgress {
|
||||
started_plan: plan,
|
||||
requested_plan: plan,
|
||||
});
|
||||
let endpoints = std::collections::HashSet::from(["disk-a".to_owned(), "disk-b".to_owned()]);
|
||||
let a = format!("{}|disk-a", uuid::Uuid::from_u128(1));
|
||||
let b = format!("{}|disk-b", uuid::Uuid::from_u128(2));
|
||||
cache.info.scan_directory_source = Some(a.clone());
|
||||
cache.activate_directory_source(b.clone(), &endpoints);
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("before first checkpoint");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist dormant linked tree");
|
||||
let previous = layout(&store).await;
|
||||
|
||||
let mut cold = DataUsageCache::default();
|
||||
cold.load(store.clone(), NAME)
|
||||
.await
|
||||
.expect("restore checkpoint in a new process");
|
||||
cold.activate_directory_source(a, &endpoints);
|
||||
assert_eq!(cold.checked_flatten("bucket").expect("first source entries").objects, 4096);
|
||||
cold.activate_directory_source(b, &endpoints);
|
||||
assert_eq!(cold.checked_flatten("bucket").expect("second source root").objects, 0);
|
||||
cold.replace(
|
||||
"bucket/new-key",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 99,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("before second checkpoint");
|
||||
cold.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save one new child");
|
||||
let after = layout(&store).await;
|
||||
let (prior_bytes, new_bytes) = page_bytes_after_save(&previous, &after);
|
||||
assert!(prior_bytes > 128 * 1024);
|
||||
assert!(
|
||||
new_bytes > 0 && new_bytes < prior_bytes / 8,
|
||||
"dormant tree rewritten: {new_bytes} new bytes for {prior_bytes} prior bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paged_checkpoint_writes_only_changed_pages_and_commits_root_last() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let first = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0)
|
||||
.await
|
||||
.expect("initial checkpoint");
|
||||
let before = layout(&store).await;
|
||||
let encoded = cache.marshal_msg().expect("encode baseline").len();
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
assert!(before[&main].1 < 8192, "only a compact manifest belongs in the root");
|
||||
assert!(before.keys().any(|key| key.contains(".pages/")), "large cache must be paged");
|
||||
|
||||
let mut restarted = DataUsageCache::default();
|
||||
let revisions = restarted
|
||||
.load_with_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("restart from durable pages");
|
||||
let prior = restarted.checked_flatten("bucket").expect("all old entries");
|
||||
restarted.replace(
|
||||
"bucket/prefix-tail",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
size: 4711,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
store.fail_puts_from.lock().await.insert(main.clone(), before[&main].0 + 1);
|
||||
restarted
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect_err("commit interrupted after writing pages");
|
||||
let interrupted = layout(&store).await;
|
||||
let mut observer = DataUsageCache::default();
|
||||
observer.load(store.clone(), NAME).await.expect("last root still readable");
|
||||
assert_eq!(observer.checked_flatten("bucket").expect("previous coverage").size, prior.size);
|
||||
store.fail_puts_from.lock().await.remove(&main);
|
||||
restarted
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("retry interrupted commit");
|
||||
observer.load(store.clone(), NAME).await.expect("load committed result");
|
||||
let actual = observer.checked_flatten("bucket").expect("new root");
|
||||
assert_eq!((actual.objects, actual.size), (4097, prior.size + 4711));
|
||||
let after = layout(&store).await;
|
||||
for (key, (writes, _)) in &interrupted {
|
||||
if key.contains(".bin.pages/") {
|
||||
assert_eq!(*writes, after[key].0, "retry restaged a previously successful immutable page");
|
||||
}
|
||||
}
|
||||
let new_bytes: usize = after
|
||||
.iter()
|
||||
.filter(|(key, _)| !before.contains_key(*key))
|
||||
.map(|(_, (_, size))| *size)
|
||||
.sum();
|
||||
assert!(
|
||||
new_bytes < encoded / 3,
|
||||
"one append staged {new_bytes} new bytes for a {encoded}-byte cache"
|
||||
);
|
||||
for (key, (writes, _)) in &before {
|
||||
if key.contains(".pages/") {
|
||||
assert_eq!(*writes, after[key].0, "a committed page was resubmitted");
|
||||
}
|
||||
}
|
||||
restarted.info.next_cycle += 1;
|
||||
let stale = restarted.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0).await;
|
||||
assert!(
|
||||
matches!(stale, Err(EcstoreError::PreconditionFailed)),
|
||||
"stale writer cannot replace the new root"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paged_checkpoint_cold_restore_spans_fixed_duration_and_repairs_corrupt_primary() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("read revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("write both copies");
|
||||
store.checkpoint_page_delay_ms.store(16, Ordering::Relaxed);
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let mut completed_at = None;
|
||||
for round in 1..=128 {
|
||||
if let Ok(result) = tokio::time::timeout(Duration::from_millis(26), loaded.load(store.clone(), NAME)).await {
|
||||
result.expect("resume exact committed manifest");
|
||||
completed_at = Some(round);
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
completed_at.is_some_and(|round| round > 1),
|
||||
"cold recovery must advance across fixed deadlines"
|
||||
);
|
||||
store.checkpoint_page_delay_ms.store(0, Ordering::Relaxed);
|
||||
let prior = loaded.checked_flatten("bucket").expect("cold cache totals");
|
||||
assert_eq!((prior.objects, prior.size), (4096, 4096 * 4097 / 2));
|
||||
|
||||
{
|
||||
let mut objects = store.objects.lock().await;
|
||||
let (_, block) = objects
|
||||
.iter_mut()
|
||||
.find(|(key, block)| key.contains(".bin.pages/") && block.len() > 8192)
|
||||
.expect("primary data page");
|
||||
block[0] ^= 0xff;
|
||||
}
|
||||
let revisions = loaded
|
||||
.load_with_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("read unaffected backup");
|
||||
assert_eq!(
|
||||
revisions,
|
||||
DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("re-enter admission on same heads")
|
||||
);
|
||||
loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("replace damaged primary generation");
|
||||
store.objects.lock().await.retain(|key, _| !key.contains(".bkp"));
|
||||
let mut primary = DataUsageCache::default();
|
||||
primary.load(store, NAME).await.expect("repaired independent primary");
|
||||
assert_eq!(primary.checked_flatten("bucket").expect("primary totals").size, prior.size);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn paged_checkpoint_bucket_admission_uses_the_scan_budget_during_cold_load() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
store.checkpoint_page_delay_ms.store(16, Ordering::Relaxed);
|
||||
|
||||
let owner = CancellationToken::new();
|
||||
let mut loaded = DataUsageCache::default();
|
||||
for round in 0..128 {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&owner,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_millis(45)),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
// Exercise the same admission used by local and remote workers. The
|
||||
// worker may drain after the deadline; only the scan token interrupts
|
||||
// recovery here, as it does in the daemon.
|
||||
let revisions = loaded
|
||||
.load_for_scan(store.clone(), NAME, &budget.token(), &budget)
|
||||
.await
|
||||
.unwrap();
|
||||
if revisions.is_some() {
|
||||
assert!(round > 0, "the first slice cannot finish this physical restore");
|
||||
assert!(!budget.token().is_cancelled(), "return work only to an active scan");
|
||||
let totals = loaded.checked_flatten("bucket").expect("restored full bucket");
|
||||
assert_eq!((totals.objects, totals.size), (4096, 4096 * 4097 / 2));
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("bucket admission repeatedly consumed the entire budget restoring the same root");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paged_checkpoint_retirement_preserves_independent_backup_and_legacy_projection() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
cache.info.snapshot_complete = true;
|
||||
let first = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("initial revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0)
|
||||
.await
|
||||
.expect("persist first generation");
|
||||
let original = layout(&store).await;
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
let backup = format!("{main}.bkp");
|
||||
let legacy =
|
||||
DataUsageCache::unmarshal(&store.objects.lock().await[&main]).expect("old reader can read compact complete root");
|
||||
let totals = legacy
|
||||
.checked_flatten_complete_scope("bucket")
|
||||
.expect("legacy compact totals");
|
||||
assert_eq!((totals.objects, totals.size), (4096, 4096 * 4097 / 2));
|
||||
let initial = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("first generation revisions");
|
||||
cache.info.next_cycle += 1;
|
||||
store
|
||||
.fail_puts_from
|
||||
.lock()
|
||||
.await
|
||||
.insert(backup.clone(), original[&backup].0 + 1);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &initial, 0)
|
||||
.await
|
||||
.expect("commit primary despite failed backup");
|
||||
let updated = layout(&store).await;
|
||||
for key in original.keys().filter(|key| key.contains(".pages/")) {
|
||||
assert_eq!(
|
||||
updated.contains_key(key),
|
||||
key.contains(".bkp.pages/"),
|
||||
"unexpected reclamation scope: {key}"
|
||||
);
|
||||
}
|
||||
let mut latest = DataUsageCache::default();
|
||||
latest.load(store.clone(), NAME).await.expect("new main remains readable");
|
||||
assert_eq!(latest.info.next_cycle, cache.info.next_cycle);
|
||||
store.objects.lock().await.remove(&main);
|
||||
latest
|
||||
.load(store, NAME)
|
||||
.await
|
||||
.expect("previous independent backup remains readable");
|
||||
assert_eq!(latest.info.next_cycle + 1, cache.info.next_cycle);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn paged_checkpoint_scan_handoff_checks_the_latest_root_before_using_its_totals() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
cache.cache.clear();
|
||||
let replace = |cache: &mut DataUsageCache, size| {
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
};
|
||||
replace(&mut cache, 17);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
store.delayed_gets.lock().await.insert(main, Duration::from_secs(6));
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
let budget = || {
|
||||
ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_secs(10)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
};
|
||||
let first = budget();
|
||||
let mut loaded = DataUsageCache::default();
|
||||
assert!(
|
||||
loaded
|
||||
.load_for_scan(store.clone(), NAME, &first.token(), &first)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(!first.token().is_cancelled(), "the slow load moved its result to the next admission");
|
||||
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
|
||||
replace(&mut cache, 99);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
let next = budget();
|
||||
assert!(
|
||||
loaded
|
||||
.load_for_scan(store, NAME, &next.token(), &next)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
assert_eq!(loaded.checked_flatten("bucket").expect("new root").size, 99);
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn indefinite_partial_checkpoints_keep_unreachable_bytes_bounded() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
let mut initial_bytes = 0;
|
||||
for round in 0..10 {
|
||||
for (name, entry) in &mut cache.cache {
|
||||
if name != "bucket" {
|
||||
entry.size += 17;
|
||||
}
|
||||
}
|
||||
// A single long-running cycle can checkpoint many times without changing cycle number.
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("current heads");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("commit partial");
|
||||
let current: usize = layout(&store)
|
||||
.await
|
||||
.iter()
|
||||
.filter(|(path, _)| path.contains(".pages/"))
|
||||
.map(|(_, (_, size))| size)
|
||||
.sum();
|
||||
if round == 0 {
|
||||
initial_bytes = current;
|
||||
assert!(initial_bytes > 100_000);
|
||||
}
|
||||
assert!(
|
||||
current <= initial_bytes * 4,
|
||||
"after {} partials, {current} page bytes for {initial_bytes} initially",
|
||||
round + 1
|
||||
);
|
||||
}
|
||||
let loaded = &mut DataUsageCache::default();
|
||||
loaded.load(store, NAME).await.expect("restore bounded checkpoint");
|
||||
assert_eq!(
|
||||
loaded.checked_flatten("bucket").expect("valid restored tree").size,
|
||||
cache.checked_flatten("bucket").expect("fixture tree").size
|
||||
);
|
||||
}
|
||||
|
||||
enum BackupDamage {
|
||||
Missing,
|
||||
Corrupt,
|
||||
UnreadableMetadata,
|
||||
}
|
||||
|
||||
async fn survives_lost_primary_after_backup_page_damage(damage: BackupDamage) {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let first = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty heads");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0)
|
||||
.await
|
||||
.expect("initial checkpoint");
|
||||
let modify_tail = |cache: &mut DataUsageCache| {
|
||||
cache.cache.get_mut("bucket/prefix-04095").expect("last leaf").size += 99;
|
||||
};
|
||||
let control = Arc::new(MemoryConfigStore::default());
|
||||
let mut target = cache.clone();
|
||||
modify_tail(&mut target);
|
||||
let empty = DataUsageCache::read_revisions(control.clone(), NAME)
|
||||
.await
|
||||
.expect("empty control store");
|
||||
target
|
||||
.save_with_revisions_for_epoch(control.clone(), NAME, &empty, 0)
|
||||
.await
|
||||
.expect("encode target in an independent fresh bundle");
|
||||
{
|
||||
let control_objects = control.objects.lock().await;
|
||||
let mut objects = store.objects.lock().await;
|
||||
let victim = objects
|
||||
.iter()
|
||||
.find_map(|(path, data)| {
|
||||
// A fresh target bundle only contains currently reachable objects.
|
||||
(path.contains(".bkp.pages/") && data.len() > 8192 && control_objects.get(path) == Some(data))
|
||||
.then(|| path.clone())
|
||||
})
|
||||
.expect("a large unchanged leaf is reachable in both manifests");
|
||||
match damage {
|
||||
BackupDamage::Corrupt => objects.get_mut(&victim).expect("existing backup leaf")[0] ^= 1,
|
||||
BackupDamage::Missing => {
|
||||
objects.remove(&victim);
|
||||
}
|
||||
BackupDamage::UnreadableMetadata => {
|
||||
drop(objects);
|
||||
store.read_errors.lock().await.insert(victim, EcstoreError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut from_main = DataUsageCache::default();
|
||||
let revisions = from_main
|
||||
.load_with_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("primary remains healthy");
|
||||
modify_tail(&mut from_main);
|
||||
from_main
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("incremental commit while primary is healthy");
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
store.objects.lock().await.remove(&main);
|
||||
let mut recovered = DataUsageCache::default();
|
||||
recovered.load(store, NAME).await.expect("standalone cold backup recovery");
|
||||
let actual = recovered
|
||||
.checked_flatten("bucket")
|
||||
.expect("a connected backup after losing its primary");
|
||||
assert_eq!((actual.objects, actual.size), (4096, 4096 * 4097 / 2 + 99));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_save_repairs_missing_backup_leaf() {
|
||||
survives_lost_primary_after_backup_page_damage(BackupDamage::Missing).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_save_repairs_corrupt_backup_leaf() {
|
||||
survives_lost_primary_after_backup_page_damage(BackupDamage::Corrupt).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_save_rebuilds_backup_with_unreadable_page_metadata() {
|
||||
survives_lost_primary_after_backup_page_damage(BackupDamage::UnreadableMetadata).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unreclaimed_partial_generation_bounds_staging_and_can_retry() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
cache.info.snapshot_complete = true;
|
||||
let empty = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty store");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &empty, 0)
|
||||
.await
|
||||
.expect("committed predecessor");
|
||||
store.fail_prefix_deletes.store(true, Ordering::Relaxed);
|
||||
cache.info.snapshot_complete = false;
|
||||
cache.info.next_cycle += 1;
|
||||
for _ in 0..8 {
|
||||
for (name, entry) in &mut cache.cache {
|
||||
if name != "bucket" {
|
||||
entry.size += 17;
|
||||
}
|
||||
}
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("current heads");
|
||||
let before = layout(&store).await;
|
||||
if let Err(error) = cache.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0).await {
|
||||
assert!(error.to_string().contains("reclaim"), "unexpected save failure: {error}");
|
||||
assert_eq!(before, layout(&store).await, "a blocked retirement must not stage further objects");
|
||||
assert!(
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
before,
|
||||
layout(&store).await,
|
||||
"retry while cleanup is unavailable must leave both generations readable"
|
||||
);
|
||||
let mut previous = DataUsageCache::default();
|
||||
previous.load(store.clone(), NAME).await.expect("last committed checkpoint");
|
||||
assert!(previous.checked_flatten("bucket").is_some());
|
||||
store.fail_prefix_deletes.store(false, Ordering::Relaxed);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save after previous retirement is reclaimed");
|
||||
previous.load(store.clone(), NAME).await.expect("read latest checkpoint");
|
||||
assert_eq!(
|
||||
cache.checked_flatten("bucket").expect("fixture").size,
|
||||
previous.checked_flatten("bucket").expect("recovered root").size
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("persistent retirement failure allowed unbounded staging");
|
||||
}
|
||||
@@ -160,6 +160,7 @@ async fn issue_8297_contended_follower_clears_only_transient_cycle_recovery() {
|
||||
ScannerMaintenanceFeatures::default(),
|
||||
None,
|
||||
runtime.startup_token(),
|
||||
&mut None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::data_usage_define::{DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCacheRevisions, DataUsageScanIdentity};
|
||||
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
const NAME: &str = "bucket/.usage-cache.bin";
|
||||
const COUNT: usize = 1536;
|
||||
|
||||
struct ReadWindow {
|
||||
pages: usize,
|
||||
device: Arc<Semaphore>,
|
||||
completed: std::sync::Mutex<Vec<String>>,
|
||||
deadline: Notify,
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static READ_WINDOW: Arc<ReadWindow>;
|
||||
}
|
||||
|
||||
// Model a serial device with one page of I/O time per admission. Completing
|
||||
// a second read would exceed the deadline, so the caller drops that waiter.
|
||||
pub(super) async fn admit_page(object: &str) -> Option<OwnedSemaphorePermit> {
|
||||
let window = READ_WINDOW.try_with(Arc::clone).ok()?;
|
||||
let permit = window.device.clone().acquire_owned().await.expect("device stays open");
|
||||
{
|
||||
let mut completed = window.completed.lock().expect("device history");
|
||||
if completed.len() < window.pages {
|
||||
completed.push(object.into());
|
||||
return Some(permit);
|
||||
}
|
||||
}
|
||||
drop(permit);
|
||||
window.deadline.notify_one();
|
||||
std::future::pending().await
|
||||
}
|
||||
|
||||
async fn window(store: &Arc<MemoryConfigStore>, loaded: &mut DataUsageCache) -> (Option<DataUsageCacheRevisions>, Vec<String>) {
|
||||
let window = Arc::new(ReadWindow {
|
||||
pages: 1,
|
||||
device: Arc::new(Semaphore::new(1)),
|
||||
completed: Default::default(),
|
||||
deadline: Notify::new(),
|
||||
});
|
||||
let result = READ_WINDOW
|
||||
.scope(window.clone(), async {
|
||||
tokio::select! {
|
||||
biased;
|
||||
value = loaded.load_with_revisions(store.clone(), NAME) => Some(value.expect("checkpoint load")),
|
||||
() = window.deadline.notified() => None,
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let pages = std::mem::take(&mut *window.completed.lock().expect("device history"));
|
||||
(result, pages)
|
||||
}
|
||||
|
||||
async fn fixture() -> (Arc<MemoryConfigStore>, DataUsageCache) {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "bucket".into();
|
||||
cache.info.source = Some(DataUsageCacheSource::new(0, 0));
|
||||
cache.info.scan_identity = Some(DataUsageScanIdentity {
|
||||
version: DATA_USAGE_SCAN_CHECKPOINT_VERSION,
|
||||
bucket_incarnation: uuid::Uuid::from_u128(9),
|
||||
set_layout: DataUsageScanPlanDigest([7; 32]),
|
||||
publication_epoch: 0,
|
||||
tier_registry_generation: 1,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
});
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
for key in 0..COUNT {
|
||||
cache.replace(
|
||||
&format!("bucket/key-{key:08}"),
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: key + 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("initial revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("seed both copies");
|
||||
(store, cache)
|
||||
}
|
||||
|
||||
async fn corrupt_primary(store: &MemoryConfigStore) {
|
||||
let mut objects = store.objects.lock().await;
|
||||
let mut pages = 0;
|
||||
for (_, data) in objects.iter_mut().filter(|(path, _)| path.contains(".bin.pages/")) {
|
||||
data[0] ^= 1;
|
||||
pages += 1;
|
||||
}
|
||||
assert!(pages > 4, "fixture must span more than one physical read batch");
|
||||
}
|
||||
|
||||
async fn finish(store: &Arc<MemoryConfigStore>, backup: bool) {
|
||||
finish_across_windows(store, backup, None).await;
|
||||
}
|
||||
|
||||
async fn finish_across_windows(store: &Arc<MemoryConfigStore>, backup: bool, competing: Option<&Arc<MemoryConfigStore>>) {
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let mut main_reads = 0;
|
||||
let mut backup_reads = 0;
|
||||
for _ in 0..128 {
|
||||
let (result, pages) = window(store, &mut loaded).await;
|
||||
for page in pages {
|
||||
if page.contains(".bkp.pages/") {
|
||||
backup_reads += 1;
|
||||
} else {
|
||||
main_reads += 1;
|
||||
}
|
||||
}
|
||||
if result.is_some() {
|
||||
let total = loaded.checked_flatten("bucket").expect("restored bucket");
|
||||
assert_eq!(total.objects, COUNT);
|
||||
assert_eq!(total.size, COUNT * (COUNT + 1) / 2);
|
||||
if backup {
|
||||
assert_eq!(main_reads, 1, "read the corrupt primary once for this revision");
|
||||
assert!(backup_reads > 4, "the independent backup must be restored across admissions");
|
||||
} else {
|
||||
assert!(main_reads > 4);
|
||||
assert_eq!(backup_reads, 0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if let Some(other) = competing {
|
||||
tokio::time::advance(Duration::from_secs(2 * 60 * 60)).await;
|
||||
let _ = window(other, &mut DataUsageCache::default()).await;
|
||||
}
|
||||
}
|
||||
panic!("checkpoint did not converge: {main_reads} primary and {backup_reads} backup page reads");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn serial_backend_restores_one_verified_page_per_deadline() {
|
||||
let (store, _) = fixture().await;
|
||||
finish(&store, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_cancellations_keep_the_longer_verified_prefix() {
|
||||
let (store, _) = fixture().await;
|
||||
let longer = Arc::new(ReadWindow {
|
||||
pages: 3,
|
||||
device: Arc::new(Semaphore::new(1)),
|
||||
completed: Default::default(),
|
||||
deadline: Notify::new(),
|
||||
});
|
||||
let mut first = DataUsageCache::default();
|
||||
let mut read = Box::pin(READ_WINDOW.scope(longer.clone(), first.load(store.clone(), NAME)));
|
||||
tokio::select! {
|
||||
result = &mut read => panic!("checkpoint unexpectedly finished within three reads: {result:?}"),
|
||||
() = longer.deadline.notified() => {}
|
||||
}
|
||||
let mut concurrent = DataUsageCache::default();
|
||||
assert!(window(&store, &mut concurrent).await.0.is_none());
|
||||
drop(read);
|
||||
|
||||
let (_, next) = window(&store, &mut concurrent).await;
|
||||
let previous = longer.completed.lock().expect("first reader history");
|
||||
assert_eq!(previous.len(), 3);
|
||||
assert!(
|
||||
!previous.contains(&next[0]),
|
||||
"a shorter concurrent cancellation displaced a verified prefix"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_primary_yields_future_admissions_to_the_healthy_backup() {
|
||||
let (store, _) = fixture().await;
|
||||
corrupt_primary(&store).await;
|
||||
finish(&store, true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_primary_revision_supersedes_an_incomplete_backup_restore() {
|
||||
let (store, _) = fixture().await;
|
||||
corrupt_primary(&store).await;
|
||||
let mut loaded = DataUsageCache::default();
|
||||
assert!(window(&store, &mut loaded).await.0.is_none());
|
||||
let (result, pages) = window(&store, &mut loaded).await;
|
||||
assert!(result.is_none());
|
||||
assert!(pages[0].contains(".bkp.pages/"), "second admission should advance backup");
|
||||
|
||||
let mut new_primary = DataUsageCache::default();
|
||||
new_primary.info.name = "bucket".into();
|
||||
new_primary.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 19,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let root = memory_config_key(RUSTFS_META_BUCKET, &format!("{}/{NAME}", crate::BUCKET_META_PREFIX));
|
||||
let bytes = rmp_serde::to_vec_named(&new_primary).expect("replacement main root");
|
||||
store.objects.lock().await.insert(root.clone(), bytes);
|
||||
*store.revisions.lock().await.get_mut(&root).expect("primary revision") += 1;
|
||||
let (result, pages) = window(&store, &mut loaded).await;
|
||||
assert!(result.is_some(), "must prefer the replacement main head immediately");
|
||||
assert!(pages.is_empty(), "must discard work on the older backup");
|
||||
let total = loaded.checked_flatten("bucket").expect("new primary root");
|
||||
assert_eq!((total.objects, total.size), (1, 19));
|
||||
}
|
||||
|
||||
async fn inject_primary_page_error(store: &MemoryConfigStore, error: EcstoreError) {
|
||||
let errors = store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.keys()
|
||||
.filter(|key| key.contains(".bin.pages/"))
|
||||
.map(|key| (key.clone(), error.clone()))
|
||||
.collect();
|
||||
*store.read_errors.lock().await = errors;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_transient_primary_page_error_does_not_schedule_the_backup() {
|
||||
let (store, _) = fixture().await;
|
||||
inject_primary_page_error(&store, EcstoreError::SlowDown).await;
|
||||
let mut loaded = DataUsageCache::default();
|
||||
assert!(window(&store, &mut loaded).await.0.is_none());
|
||||
store.read_errors.lock().await.clear();
|
||||
finish(&store, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_phase_is_bound_to_its_storage_instance() {
|
||||
let (damaged, _) = fixture().await;
|
||||
corrupt_primary(&damaged).await;
|
||||
assert!(window(&damaged, &mut DataUsageCache::default()).await.0.is_none());
|
||||
|
||||
let (healthy, _) = fixture().await;
|
||||
let first = DataUsageCache::read_revisions(damaged.clone(), NAME)
|
||||
.await
|
||||
.expect("failed primary revision");
|
||||
let second = DataUsageCache::read_revisions(healthy.clone(), NAME)
|
||||
.await
|
||||
.expect("other storage revision");
|
||||
assert_eq!(first, second, "both stores must expose the same ETags");
|
||||
finish(&healthy, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_restore_keeps_verified_pages_until_another_bucket_needs_its_slot() {
|
||||
let (primary, _) = fixture().await;
|
||||
let (other, _) = fixture().await;
|
||||
finish_across_windows(&primary, false, Some(&other)).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_restore_keeps_the_failed_primary_revision_across_scheduled_rounds() {
|
||||
let (primary, _) = fixture().await;
|
||||
let (other, _) = fixture().await;
|
||||
corrupt_primary(&primary).await;
|
||||
corrupt_primary(&other).await;
|
||||
finish_across_windows(&primary, true, Some(&other)).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn physical_primary_page_corruption_falls_back_to_the_healthy_copy() {
|
||||
let (store, _) = fixture().await;
|
||||
inject_primary_page_error(&store, EcstoreError::FileCorrupt).await;
|
||||
finish(&store, true).await;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,714 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Cycle-local execution can borrow an interrupted physical directory stream.
|
||||
//! Only the current scanner owns admission and classification; parked streams
|
||||
//! have no task, budget or authority to update a bucket cache.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
mod checkpoint;
|
||||
mod pages;
|
||||
pub use checkpoint::DirectoryCheckpoint;
|
||||
pub(super) use checkpoint::Saved;
|
||||
pub(crate) use checkpoint::{DirectoryProgress, DirectoryReadProgress};
|
||||
use checkpoint::{Physical, Recorder, Scope};
|
||||
use pages::NamePages;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, LazyLock, Mutex, Weak};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use super::{DirectoryStream, Disk, FolderScanMode, ScannerBucketPrefixScanScope, native_io};
|
||||
use crate::data_usage_define::{DataUsageCacheInfo, DataUsageCacheSource, DataUsageScanIdentity, DataUsageScanProgress};
|
||||
use crate::scanner_budget::ScannerCycleBudget;
|
||||
|
||||
pub(super) const PARKED_DIRECTORIES_PER_DISK: usize = 32;
|
||||
|
||||
/// Only a positively identified disk can re-adopt a linked tree. A missing
|
||||
/// format on an otherwise visible local drive uses its directory generation;
|
||||
/// genuine format I/O errors propagate before any cached subtotals are used.
|
||||
pub(super) async fn physical_source(disk: &Arc<Disk>) -> Result<String, super::ScannerError> {
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
use sha2::{Digest, Sha256};
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
let generation = match disk.get_disk_id().await {
|
||||
Ok(Some(id)) if !id.is_nil() => id.to_string(),
|
||||
Ok(None) | Err(super::DiskError::UnformattedDisk) => {
|
||||
let path = disk.path();
|
||||
let stamp = DirectoryStamp::at(path.to_str().ok_or_else(|| io::Error::from(io::ErrorKind::InvalidInput))?).await?;
|
||||
// Bucket creation and scanner startup can change the volume root's
|
||||
// mtime/ctime. Birth time distinguishes inode reuse without making
|
||||
// ordinary directory content changes appear to replace the disk.
|
||||
if let Some(generation) = stamp.volume_generation() {
|
||||
let bytes = rmp_serde::to_vec_named(&(std::env::consts::OS, generation)).map_err(io::Error::other)?;
|
||||
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
|
||||
} else {
|
||||
// The platform cannot attest the generation across processes.
|
||||
uuid::Uuid::new_v4().to_string()
|
||||
}
|
||||
}
|
||||
Ok(Some(_)) => return Err(io::Error::new(io::ErrorKind::InvalidData, "scanner disk format contains a nil ID").into()),
|
||||
Err(error) => return Err(io::Error::other(error).into()),
|
||||
};
|
||||
Ok(format!("{generation}|{endpoint}"))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct DirectoryScope {
|
||||
disk: Weak<Disk>,
|
||||
bucket: Arc<str>,
|
||||
source: DataUsageCacheSource,
|
||||
identity: DataUsageScanIdentity,
|
||||
progress: DataUsageScanProgress,
|
||||
mode: FolderScanMode,
|
||||
prefix_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
owner: (u64, u64),
|
||||
endpoint: String,
|
||||
watch: Option<[u8; 32]>,
|
||||
continuing: bool,
|
||||
}
|
||||
|
||||
impl DirectoryScope {
|
||||
pub(super) fn new(
|
||||
disk: &Arc<Disk>,
|
||||
cache: &super::DataUsageCache,
|
||||
mode: FolderScanMode,
|
||||
prefix_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
) -> Option<Self> {
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
let info = &cache.info;
|
||||
Some(Self {
|
||||
disk: Arc::downgrade(disk),
|
||||
bucket: Arc::from(info.name.as_str()),
|
||||
source: info.source?,
|
||||
identity: info.scan_identity.filter(DataUsageScanIdentity::is_valid)?,
|
||||
progress: info.scan_progress?,
|
||||
mode,
|
||||
prefix_scope,
|
||||
owner: (info.leader_epoch, info.next_cycle),
|
||||
endpoint: format!("|{}", disk.endpoint()),
|
||||
watch: info
|
||||
.scan_usage_watch
|
||||
.as_ref()
|
||||
.filter(|proof| proof.is_well_formed())
|
||||
.map(|proof| proof.digest().0),
|
||||
continuing: cache.has_scan_position(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn bind_source(&mut self, source: Option<&str>) {
|
||||
if let Some(source) = source {
|
||||
self.endpoint = source.to_owned();
|
||||
}
|
||||
}
|
||||
pub(super) fn endpoint(&self) -> &str {
|
||||
&self.endpoint
|
||||
}
|
||||
fn matches(&self, other: &Self) -> bool {
|
||||
self.endpoint == other.endpoint
|
||||
&& self.disk.ptr_eq(&other.disk)
|
||||
&& self.bucket == other.bucket
|
||||
&& self.source == other.source
|
||||
&& self.identity == other.identity
|
||||
&& self.progress == other.progress
|
||||
&& self.mode == other.mode
|
||||
&& self.watch == other.watch
|
||||
&& self.prefix_scope == other.prefix_scope
|
||||
&& (other.continuing || self.owner == other.owner)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq)]
|
||||
struct DirectoryStamp {
|
||||
modified: SystemTime,
|
||||
birth: Option<SystemTime>,
|
||||
#[cfg(unix)]
|
||||
device: u64,
|
||||
#[cfg(unix)]
|
||||
inode: u64,
|
||||
#[cfg(unix)]
|
||||
changed: (i64, i64),
|
||||
#[cfg(windows)]
|
||||
identity: same_file::Handle,
|
||||
#[cfg(windows)]
|
||||
physical: Option<(u64, u64, i64)>,
|
||||
}
|
||||
|
||||
impl DirectoryStamp {
|
||||
fn volume_generation(&self) -> Option<(u64, u64, SystemTime)> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
Some((self.device, self.inode, self.birth?))
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let (volume, file, _) = self.physical?;
|
||||
Some((volume, file, self.birth?))
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
async fn at(path: &str) -> io::Result<Self> {
|
||||
let path = PathBuf::from(path);
|
||||
crate::storage_api::ecstore_spawn_blocking_with_resource_ownership(move || {
|
||||
let meta = std::fs::symlink_metadata(&path)?;
|
||||
if !meta.is_dir() {
|
||||
return Err(io::ErrorKind::NotADirectory.into());
|
||||
}
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
#[cfg(windows)]
|
||||
let identity = same_file::Handle::from_path(&path)?;
|
||||
#[cfg(windows)]
|
||||
let physical = winapi_util::file::information(identity.as_file()).ok().and_then(|info| {
|
||||
let created = i64::try_from(info.creation_time()?).ok()?;
|
||||
(info.file_index() != 0).then_some((info.volume_serial_number(), info.file_index(), created))
|
||||
});
|
||||
Ok(Self {
|
||||
modified: meta.modified()?,
|
||||
birth: meta.created().ok(),
|
||||
#[cfg(unix)]
|
||||
device: meta.dev(),
|
||||
#[cfg(unix)]
|
||||
inode: meta.ino(),
|
||||
#[cfg(unix)]
|
||||
changed: (meta.ctime(), meta.ctime_nsec()),
|
||||
#[cfg(windows)]
|
||||
identity,
|
||||
#[cfg(windows)]
|
||||
physical,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(io::Error::other)?
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct DirectoryWork {
|
||||
names: NamePages,
|
||||
// For a prefix tree the canonical child values live in the bucket cache.
|
||||
// The final compaction threshold caps how many paths this list can hold.
|
||||
pub(super) linked: Vec<String>,
|
||||
pub(super) compacted_total: super::DataUsageEntry,
|
||||
pub(super) gap: bool,
|
||||
compacted: bool,
|
||||
recorder: Option<Recorder>,
|
||||
}
|
||||
|
||||
impl DirectoryWork {
|
||||
pub(super) fn restore_mode(&mut self, compacted: bool) {
|
||||
if self.compacted && !compacted {
|
||||
self.rewind();
|
||||
}
|
||||
}
|
||||
fn maybe_checkpoint(&mut self) {
|
||||
if let Some(recorder) = &mut self.recorder
|
||||
&& recorder.last_update.elapsed() >= Duration::from_secs(1)
|
||||
{
|
||||
recorder.last_update = Instant::now();
|
||||
self.checkpoint();
|
||||
}
|
||||
}
|
||||
pub(super) fn checkpoint(&self) {
|
||||
if let Some(recorder) = &self.recorder {
|
||||
recorder.save(self);
|
||||
}
|
||||
}
|
||||
pub(super) async fn next_name(&mut self) -> io::Result<Option<String>> {
|
||||
self.names.next().await
|
||||
}
|
||||
pub(super) fn acknowledge(&mut self, name: &str, linked: bool, compacted: bool) {
|
||||
if self.names.acknowledge() {
|
||||
self.compacted = compacted;
|
||||
if linked && !compacted {
|
||||
self.linked.push(name.to_owned());
|
||||
}
|
||||
self.maybe_checkpoint();
|
||||
}
|
||||
}
|
||||
pub(super) fn rewind(&mut self) {
|
||||
self.compacted = false;
|
||||
self.names.rewind();
|
||||
self.linked.clear();
|
||||
self.compacted_total = super::DataUsageEntry::default();
|
||||
self.gap = false;
|
||||
}
|
||||
}
|
||||
|
||||
struct Listing {
|
||||
reader: Option<DirectoryStream>,
|
||||
pending: Option<native_io::DirEntry>,
|
||||
delivery: Option<(native_io::DirEntry, bool)>,
|
||||
classified: Option<Option<std::fs::FileType>>,
|
||||
markers: [Option<native_io::DirEntry>; 2],
|
||||
abandoned: Option<super::DataUsageHashMap>,
|
||||
work: DirectoryWork,
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
struct Parked {
|
||||
scope: DirectoryScope,
|
||||
parked_at: Instant,
|
||||
serial: u64,
|
||||
stamp: DirectoryStamp,
|
||||
listing: Box<Listing>,
|
||||
}
|
||||
|
||||
static PARKED: LazyLock<Mutex<HashMap<PathBuf, Parked>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
fn expired_slots(slots: &mut HashMap<PathBuf, Parked>) -> Vec<(PathBuf, Parked)> {
|
||||
slots
|
||||
.extract_if(|_, parked| {
|
||||
parked.scope.disk.strong_count() == 0 || parked.parked_at.elapsed() > Duration::from_secs(24 * 3600)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A routing hint only. The admitted walker revalidates the physical source,
|
||||
/// bucket instance, coverage plan and directory before using a parked stream.
|
||||
pub(crate) fn parked_bucket_disks(bucket: &str) -> Vec<Weak<Disk>> {
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let expired = expired_slots(&mut slots);
|
||||
let matches = slots
|
||||
.values()
|
||||
.filter(|entry| entry.scope.bucket.as_ref() == bucket)
|
||||
.map(|entry| entry.scope.disk.clone())
|
||||
.collect();
|
||||
drop(slots);
|
||||
drop(expired);
|
||||
matches
|
||||
}
|
||||
|
||||
pub(super) fn is_parked(path: &str) -> bool {
|
||||
PARKED.lock().unwrap_or_else(|p| p.into_inner()).contains_key(Path::new(path))
|
||||
}
|
||||
|
||||
pub(super) fn forget_bucket(disk: &Arc<Disk>, info: &DataUsageCacheInfo) {
|
||||
let owner = Arc::downgrade(disk);
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let removed = slots
|
||||
.extract_if(|_, parked| {
|
||||
parked.scope.disk.ptr_eq(&owner)
|
||||
&& parked.scope.bucket.as_ref() == info.name
|
||||
&& parked.scope.owner <= (info.leader_epoch, info.next_cycle)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
drop(slots);
|
||||
drop(removed);
|
||||
}
|
||||
|
||||
// Prefer existing active work over cyclic LRU eviction. Under pressure, a
|
||||
// deleted or replaced directory must not monopolize one of the physical slots.
|
||||
async fn reclaim_retired(scope: &DirectoryScope, path: &Path) {
|
||||
static NEXT_PROBE: AtomicUsize = AtomicUsize::new(0);
|
||||
let candidate = {
|
||||
let slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
if slots.contains_key(path) {
|
||||
return;
|
||||
}
|
||||
let disk_slots: Vec<_> = slots
|
||||
.iter()
|
||||
.filter(|(_, parked)| parked.scope.disk.ptr_eq(&scope.disk))
|
||||
.collect();
|
||||
if disk_slots.len() < PARKED_DIRECTORIES_PER_DISK {
|
||||
return;
|
||||
}
|
||||
let (path, parked) = disk_slots[NEXT_PROBE.fetch_add(1, Ordering::Relaxed) % disk_slots.len()];
|
||||
(path.clone(), parked.serial)
|
||||
};
|
||||
let Some(candidate_path) = candidate.0.to_str() else { return };
|
||||
let current = DirectoryStamp::at(candidate_path).await;
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let stale = slots.get(&candidate.0).is_some_and(|parked| {
|
||||
parked.serial == candidate.1
|
||||
&& match ¤t {
|
||||
Ok(stamp) => stamp != &parked.stamp,
|
||||
Err(error) => matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory),
|
||||
}
|
||||
});
|
||||
let retired = stale.then(|| slots.remove(&candidate.0));
|
||||
drop(slots);
|
||||
drop(retired);
|
||||
}
|
||||
|
||||
fn valid_component(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.len() <= 4096
|
||||
&& name != "."
|
||||
&& name != ".."
|
||||
&& !name.as_bytes().contains(&0)
|
||||
&& std::path::Path::new(name)
|
||||
.file_name()
|
||||
.is_some_and(|part| part == std::ffi::OsStr::new(name))
|
||||
}
|
||||
|
||||
pub(super) struct ScannerDirectory {
|
||||
path: Arc<Path>,
|
||||
scope: Option<(DirectoryScope, DirectoryStamp)>,
|
||||
directory_name: Arc<str>,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
listing: Option<Box<Listing>>,
|
||||
marker: usize,
|
||||
pub(super) enumeration_only: bool,
|
||||
resumed: bool,
|
||||
pub(super) directory_replaced: bool,
|
||||
pub(super) batch_entries: usize,
|
||||
}
|
||||
|
||||
impl ScannerDirectory {
|
||||
pub(super) async fn open(
|
||||
path: &str,
|
||||
directory_name: &str,
|
||||
scope: Option<DirectoryScope>,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
saved: Option<Saved>,
|
||||
) -> io::Result<Self> {
|
||||
// A different physical directory must never inherit old names. Directory
|
||||
// timestamps supplement (and cannot replace) the scanner's plan fences.
|
||||
let stamp = if scope.is_some() {
|
||||
Some(DirectoryStamp::at(path).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(scope) = &scope {
|
||||
reclaim_retired(scope, Path::new(path)).await;
|
||||
}
|
||||
let parked = scope.as_ref().and_then(|current| {
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let key = Path::new(path);
|
||||
if slots
|
||||
.get(key)
|
||||
.is_some_and(|p| p.scope.disk.ptr_eq(¤t.disk) && p.scope.owner > current.owner)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
slots.remove(key)
|
||||
});
|
||||
let directory_replaced = parked.as_ref().is_some_and(|entry| Some(&entry.stamp) != stamp.as_ref());
|
||||
let parked =
|
||||
parked.filter(|entry| !directory_replaced && scope.as_ref().is_some_and(|current| entry.scope.matches(current)));
|
||||
let mut resumed = parked.is_some();
|
||||
let mut previous = saved
|
||||
.as_ref()
|
||||
.and_then(|slot| slot.lock().unwrap_or_else(|p| p.into_inner()).clone());
|
||||
let durable_scope = scope.as_ref().map(Scope::new);
|
||||
let endpoint = scope.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default();
|
||||
let physical = scope
|
||||
.as_ref()
|
||||
.zip(stamp.as_ref())
|
||||
.and_then(|(scope, stamp)| Physical::of(&scope.endpoint, stamp));
|
||||
let mut replaced = directory_replaced
|
||||
|| previous
|
||||
.as_ref()
|
||||
.is_some_and(|old| old.endpoint() == endpoint && old.physical.is_some() && old.physical != physical);
|
||||
previous = previous.filter(|old| {
|
||||
durable_scope
|
||||
.as_ref()
|
||||
.is_some_and(|current| old.restore_allowed(directory_name, current, &physical))
|
||||
});
|
||||
let mut listing = if let Some(parked) = parked {
|
||||
parked.listing
|
||||
} else {
|
||||
let mut reader = None;
|
||||
if let Some(checkpoint) = &previous {
|
||||
if checkpoint.sealed {
|
||||
resumed = true;
|
||||
} else if let Some(anchor) = &checkpoint.anchor {
|
||||
match DirectoryStream::open_at(path, anchor).await {
|
||||
Ok(restored) => {
|
||||
reader = Some(restored);
|
||||
resumed = true;
|
||||
}
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
io::ErrorKind::InvalidData
|
||||
| io::ErrorKind::Unsupported
|
||||
| io::ErrorKind::NotFound
|
||||
| io::ErrorKind::InvalidInput
|
||||
) =>
|
||||
{
|
||||
previous = None;
|
||||
replaced = true;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
if !resumed {
|
||||
reader = Some(DirectoryStream::open(path, scope.is_some()).await?);
|
||||
}
|
||||
let mut work = DirectoryWork::default();
|
||||
let mut markers = [None, None];
|
||||
if let Some(saved) = &previous {
|
||||
if resumed && let Some(owner) = &scope {
|
||||
tracing::debug!(
|
||||
event = super::EVENT_SCANNER_FOLDER_STATE,
|
||||
component = super::LOG_COMPONENT_SCANNER,
|
||||
subsystem = super::LOG_SUBSYSTEM_FOLDER,
|
||||
state = "directory_checkpoint_restored",
|
||||
bucket = %owner.bucket,
|
||||
directory = directory_name,
|
||||
disk = %endpoint,
|
||||
leader_epoch = owner.owner.0,
|
||||
cycle = owner.owner.1,
|
||||
restore_mode = if saved.sealed { "sealed_membership" } else { "native_position" },
|
||||
entries_read = saved.entries_read,
|
||||
completed_through = saved.aggregate.as_ref().map(|aggregate| aggregate.through.as_str()),
|
||||
"Restored directory checkpoint"
|
||||
);
|
||||
}
|
||||
work.names = NamePages::restore(&saved.pages, saved.sealed, saved.aggregate.as_ref().map(|a| a.through.as_str()));
|
||||
if let Some(aggregate) = &saved.aggregate {
|
||||
work.compacted = true;
|
||||
work.compacted_total = aggregate.value.clone();
|
||||
}
|
||||
let physical_path: Arc<Path> = Arc::from(Path::new(path));
|
||||
if let Some(part) = &saved.part {
|
||||
markers[0] = Some(native_io::DirEntry::recorded(physical_path.clone(), part.into(), None));
|
||||
}
|
||||
if saved.metadata {
|
||||
markers[1] = Some(native_io::DirEntry::recorded(physical_path, super::STORAGE_FORMAT_FILE.into(), None));
|
||||
}
|
||||
}
|
||||
Box::new(Listing {
|
||||
reader,
|
||||
pending: None,
|
||||
delivery: None,
|
||||
classified: None,
|
||||
markers,
|
||||
abandoned: None,
|
||||
work,
|
||||
failed: false,
|
||||
})
|
||||
};
|
||||
if let (Some(slot), Some(durable_scope)) = (saved, durable_scope) {
|
||||
if let Some(recorder) = &mut listing.work.recorder {
|
||||
recorder.slot = slot;
|
||||
} else {
|
||||
let recorder = Recorder::new(slot, directory_name, durable_scope, endpoint, physical, previous.as_ref());
|
||||
|
||||
listing.work.recorder = Some(recorder);
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
path: Arc::from(Path::new(path)),
|
||||
scope: scope.zip(stamp),
|
||||
directory_name: Arc::from(directory_name),
|
||||
budget,
|
||||
listing: Some(listing),
|
||||
marker: 0,
|
||||
enumeration_only: false,
|
||||
resumed,
|
||||
directory_replaced: replaced,
|
||||
batch_entries: 0,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn continues_stream(&self) -> bool {
|
||||
self.resumed
|
||||
}
|
||||
pub(super) fn indexed(&self) -> bool {
|
||||
self.scope.is_some()
|
||||
}
|
||||
pub(super) async fn verify_current(&mut self) -> io::Result<()> {
|
||||
let Some((_, original)) = &self.scope else { return Ok(()) };
|
||||
let path = self.path.to_str().ok_or(io::ErrorKind::InvalidInput)?;
|
||||
let current = DirectoryStamp::at(path).await?;
|
||||
if *original == current {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(listing) = &mut self.listing {
|
||||
listing.failed = true;
|
||||
}
|
||||
Err(io::Error::new(io::ErrorKind::Interrupted, "scanner directory changed during traversal"))
|
||||
}
|
||||
|
||||
pub(super) fn folder_count(&self) -> Option<usize> {
|
||||
self.indexed()
|
||||
.then(|| self.listing.as_ref().map_or(0, |l| l.work.names.len()))
|
||||
}
|
||||
pub(super) fn finished_listing(&self) -> bool {
|
||||
self.listing.as_ref().is_some_and(|l| l.work.names.ready())
|
||||
}
|
||||
pub(super) fn work(&mut self) -> Option<&mut DirectoryWork> {
|
||||
if !self.indexed() {
|
||||
return None;
|
||||
}
|
||||
self.listing.as_mut().map(|listing| &mut listing.work)
|
||||
}
|
||||
pub(super) fn resolve_abandoned(&mut self, path: &str) {
|
||||
if let Some(abandoned) = self.listing.as_mut().and_then(|l| l.abandoned.as_mut()) {
|
||||
abandoned.remove(path);
|
||||
}
|
||||
}
|
||||
pub(super) fn hold_abandoned(&mut self, baseline: &mut super::DataUsageHashMap) {
|
||||
if self.indexed()
|
||||
&& let Some(listing) = &mut self.listing
|
||||
&& listing.abandoned.is_none()
|
||||
{
|
||||
if !baseline.is_empty() && self.resumed {
|
||||
// Cold restore carries classified members, not the previous
|
||||
// process's transient unresolved heal candidates.
|
||||
for page in listing.work.names.snapshot() {
|
||||
for name in page.names() {
|
||||
baseline.remove(&super::path_join_buf(&[&self.directory_name, name]));
|
||||
}
|
||||
}
|
||||
}
|
||||
listing.abandoned = Some(std::mem::take(baseline));
|
||||
}
|
||||
}
|
||||
pub(super) fn restore_abandoned(&self, baseline: &mut super::DataUsageHashMap) {
|
||||
if let Some(abandoned) = self.listing.as_ref().and_then(|listing| listing.abandoned.as_ref()) {
|
||||
baseline.clone_from(abandoned);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn peek_entry(&mut self) -> io::Result<Option<native_io::DirEntry>> {
|
||||
if let Some((entry, enumeration_only)) = self.listing.as_ref().and_then(|listing| listing.delivery.as_ref()) {
|
||||
self.enumeration_only = *enumeration_only;
|
||||
return Ok(Some(entry.clone()));
|
||||
}
|
||||
let entry = self.next_entry().await?;
|
||||
if let (Some(listing), Some(entry)) = (&mut self.listing, &entry) {
|
||||
listing.delivery = Some((entry.clone(), self.enumeration_only));
|
||||
}
|
||||
Ok(entry)
|
||||
}
|
||||
|
||||
pub(super) fn accept_entry(&mut self) {
|
||||
let Some(listing) = &mut self.listing else { return };
|
||||
let Some((entry, _)) = listing.delivery.take() else { return };
|
||||
// Discovering a name can precede the I/O admission wait. Commit its
|
||||
// classification and native position together only after admission;
|
||||
// a cold retry then re-reads any name still pending on cancellation.
|
||||
let Some(kind) = listing.classified.take() else { return };
|
||||
let name = entry.file_name();
|
||||
let text = name.to_string_lossy();
|
||||
if let Some(state) = &mut listing.work.recorder {
|
||||
state.anchor = entry.anchor().cloned();
|
||||
state.entries_read = state.entries_read.saturating_add(1);
|
||||
state.valid_names &= name.to_str().is_some_and(valid_component);
|
||||
}
|
||||
if let Some(kind) = kind {
|
||||
let direct_part = kind.is_file() && super::is_erasure_part_name(&text);
|
||||
if kind.is_dir() {
|
||||
if let Some(abandoned) = &mut listing.abandoned
|
||||
&& !abandoned.is_empty()
|
||||
{
|
||||
abandoned.remove(&super::path_join_buf(&[&self.directory_name, &text]));
|
||||
}
|
||||
listing.work.names.add(text.into_owned());
|
||||
} else if direct_part {
|
||||
if let Some(state) = &mut listing.work.recorder {
|
||||
state.part.get_or_insert_with(|| text.to_string());
|
||||
}
|
||||
listing.markers[0].get_or_insert_with(|| native_io::DirEntry::recorded(self.path.clone(), name, Some(kind)));
|
||||
} else if text == super::STORAGE_FORMAT_FILE {
|
||||
if let Some(state) = &mut listing.work.recorder {
|
||||
state.metadata = true;
|
||||
}
|
||||
listing.markers[1] = Some(native_io::DirEntry::recorded(self.path.clone(), name, Some(kind)));
|
||||
}
|
||||
}
|
||||
listing.work.maybe_checkpoint();
|
||||
}
|
||||
|
||||
async fn next_entry(&mut self) -> io::Result<Option<native_io::DirEntry>> {
|
||||
let erasure_candidate = self
|
||||
.scope
|
||||
.as_ref()
|
||||
.is_some_and(|(scope, _)| scope.mode != FolderScanMode::Namespace);
|
||||
let indexed = self.indexed();
|
||||
let listing = self
|
||||
.listing
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::other("scanner directory already returned"))?;
|
||||
self.enumeration_only = false;
|
||||
if let Some(reader) = listing.reader.as_mut() {
|
||||
if !indexed {
|
||||
return reader.next_entry().await;
|
||||
}
|
||||
listing.work.names.flush_full().await?;
|
||||
if listing.pending.is_none() {
|
||||
listing.pending = match reader.next_entry().await {
|
||||
Ok(entry) => entry,
|
||||
Err(error) => {
|
||||
listing.failed = true;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
}
|
||||
if let Some(entry) = &listing.pending {
|
||||
let kind = match entry.file_type().await {
|
||||
Ok(kind) => Some(kind),
|
||||
Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::TooManyLinks) => None,
|
||||
Err(error) => {
|
||||
listing.failed = true;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let name = entry.file_name();
|
||||
let text = name.to_string_lossy();
|
||||
self.enumeration_only = kind.is_none_or(|kind| {
|
||||
(!kind.is_symlink() || text == super::STORAGE_FORMAT_FILE)
|
||||
&& !(erasure_candidate && kind.is_file() && super::is_erasure_part_name(&text))
|
||||
});
|
||||
listing.classified = Some(kind);
|
||||
return Ok(listing.pending.take());
|
||||
}
|
||||
listing.reader = None;
|
||||
listing.work.names.finish();
|
||||
listing.work.maybe_checkpoint();
|
||||
}
|
||||
while self.marker < listing.markers.len() {
|
||||
let marker = self.marker;
|
||||
self.marker += 1;
|
||||
if let Some(entry) = &listing.markers[marker] {
|
||||
return Ok(Some(entry.clone()));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScannerDirectory {
|
||||
fn drop(&mut self) {
|
||||
let Some((scope, stamp)) = self.scope.take() else { return };
|
||||
let Some(listing) = self.listing.take() else { return };
|
||||
if listing.failed || self.budget.reason().is_none() {
|
||||
if let Some(recorder) = &listing.work.recorder {
|
||||
*recorder.slot.lock().unwrap_or_else(|p| p.into_inner()) = None;
|
||||
}
|
||||
return;
|
||||
}
|
||||
listing.work.checkpoint();
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let expired = expired_slots(&mut slots);
|
||||
let key = self.path.to_path_buf();
|
||||
let replaced = slots
|
||||
.get(&key)
|
||||
.is_some_and(|p| !p.scope.disk.ptr_eq(&scope.disk) || p.scope.owner < scope.owner)
|
||||
.then(|| slots.remove(&key));
|
||||
if slots.values().filter(|p| p.scope.disk.ptr_eq(&scope.disk)).count() < PARKED_DIRECTORIES_PER_DISK
|
||||
&& let std::collections::hash_map::Entry::Vacant(slot) = slots.entry(key)
|
||||
{
|
||||
static NEXT_SERIAL: AtomicU64 = AtomicU64::new(0);
|
||||
slot.insert(Parked {
|
||||
scope,
|
||||
parked_at: Instant::now(),
|
||||
serial: NEXT_SERIAL.fetch_add(1, Ordering::Relaxed),
|
||||
stamp,
|
||||
listing,
|
||||
});
|
||||
}
|
||||
drop(slots);
|
||||
drop((expired, replaced));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Instant, SystemTime};
|
||||
|
||||
use super::{DirectoryScope, DirectoryStamp, DirectoryWork, FolderScanMode, native_io, pages::NamePage};
|
||||
use crate::data_usage_define::{
|
||||
DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageScanIdentity, DataUsageScanProgress,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(super) struct Scope {
|
||||
bucket: String,
|
||||
source: DataUsageCacheSource,
|
||||
identity: DataUsageScanIdentity,
|
||||
progress: DataUsageScanProgress,
|
||||
mode: u8,
|
||||
prefixes: Option<[u8; 32]>,
|
||||
watch: Option<[u8; 32]>,
|
||||
}
|
||||
|
||||
impl Scope {
|
||||
fn may_adopt(&self, current: &Self) -> bool {
|
||||
self.bucket == current.bucket
|
||||
&& self.source == current.source
|
||||
&& self.identity == current.identity
|
||||
&& self.mode == current.mode
|
||||
&& self.prefixes == current.prefixes
|
||||
&& self.progress.started_plan == current.progress.started_plan
|
||||
&& ((self.progress == current.progress && self.watch == current.watch)
|
||||
|| current.progress.started_plan != current.progress.requested_plan)
|
||||
}
|
||||
pub(super) fn new(owner: &DirectoryScope) -> Self {
|
||||
let mode = match owner.mode {
|
||||
FolderScanMode::Namespace => 0,
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace,
|
||||
parent_has_metadata,
|
||||
} => 1 + u8::from(reuse_namespace) + 2 * u8::from(parent_has_metadata),
|
||||
};
|
||||
let prefixes = owner.prefix_scope.as_ref().map(|scope| {
|
||||
let mut entries: Vec<_> = scope.selected_top_level_entries.iter().collect();
|
||||
entries.sort_unstable();
|
||||
let mut hash = Sha256::new();
|
||||
for entry in entries {
|
||||
hash.update(u64::try_from(entry.len()).unwrap_or(u64::MAX).to_le_bytes());
|
||||
hash.update(entry.as_bytes());
|
||||
}
|
||||
hash.finalize().into()
|
||||
});
|
||||
Self {
|
||||
bucket: owner.bucket.to_string(),
|
||||
source: owner.source,
|
||||
identity: owner.identity,
|
||||
progress: owner.progress,
|
||||
mode,
|
||||
prefixes,
|
||||
watch: owner.watch,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct Physical {
|
||||
pub(crate) endpoint: String,
|
||||
device: u64,
|
||||
inode: u64,
|
||||
changed: (i64, i64),
|
||||
modified: SystemTime,
|
||||
}
|
||||
|
||||
impl Physical {
|
||||
#[cfg(unix)]
|
||||
pub(super) fn of(endpoint: &str, stamp: &DirectoryStamp) -> Option<Self> {
|
||||
Some(Self {
|
||||
endpoint: endpoint.to_owned(),
|
||||
device: stamp.device,
|
||||
inode: stamp.inode,
|
||||
changed: stamp.changed,
|
||||
modified: stamp.modified,
|
||||
})
|
||||
}
|
||||
#[cfg(windows)]
|
||||
pub(super) fn of(endpoint: &str, stamp: &DirectoryStamp) -> Option<Self> {
|
||||
let (device, inode, created) = stamp.physical?;
|
||||
Some(Self {
|
||||
endpoint: endpoint.to_owned(),
|
||||
device,
|
||||
inode,
|
||||
changed: (created, 0),
|
||||
modified: stamp.modified,
|
||||
})
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
pub(super) fn of(_endpoint: &str, _stamp: &DirectoryStamp) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub(super) struct Aggregate {
|
||||
pub(super) through: String,
|
||||
pub(super) value: DataUsageEntry,
|
||||
}
|
||||
|
||||
/// Immutable directory membership and the last fully counted compacted child.
|
||||
/// The active child's provisional contribution is never part of this receipt.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct DirectoryCheckpoint {
|
||||
version: u8,
|
||||
pub(super) parent: String,
|
||||
scope: Scope,
|
||||
pub(super) physical: Option<Physical>,
|
||||
pub(super) pages: Vec<NamePage>,
|
||||
pub(super) anchor: Option<native_io::DirectoryAnchor>,
|
||||
pub(super) entries_read: u64,
|
||||
pub(super) sealed: bool,
|
||||
pub(super) part: Option<String>,
|
||||
pub(super) metadata: bool,
|
||||
pub(super) aggregate: Option<Aggregate>,
|
||||
endpoint: String,
|
||||
digest: [u8; 32],
|
||||
}
|
||||
|
||||
/// Physical progress orders only callbacks from the same directory generation.
|
||||
/// The coordinator uses these receipts for fair scheduling, never for coverage.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DirectoryReadProgress {
|
||||
pub(crate) parent: String,
|
||||
pub(crate) physical: Physical,
|
||||
position: (u64, bool),
|
||||
}
|
||||
impl DirectoryReadProgress {
|
||||
pub(crate) fn same_source(&self, other: &Self) -> bool {
|
||||
self.parent == other.parent && self.physical == other.physical
|
||||
}
|
||||
pub(crate) fn extends(&self, older: &Self) -> bool {
|
||||
self.same_source(older) && self.position >= older.position
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DirectoryProgress {
|
||||
pub(crate) parent: String,
|
||||
pub(crate) physical: Physical,
|
||||
manifest: [u8; 32],
|
||||
through: String,
|
||||
}
|
||||
impl DirectoryProgress {
|
||||
pub(crate) fn same_source(&self, previous: &Self) -> bool {
|
||||
self.parent == previous.parent && self.physical == previous.physical && self.manifest == previous.manifest
|
||||
}
|
||||
pub(crate) fn extends(&self, previous: &Self) -> bool {
|
||||
self.same_source(previous) && self.through >= previous.through
|
||||
}
|
||||
pub(crate) fn covers(&self, path: &str) -> bool {
|
||||
path.strip_prefix(&self.parent)
|
||||
.and_then(|tail| tail.strip_prefix('/'))
|
||||
.and_then(|tail| tail.split('/').next())
|
||||
.is_some_and(|child| child <= self.through.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl DirectoryCheckpoint {
|
||||
/// Deserialize each replica's independent receipt, then intern equal
|
||||
/// verified immutable name pages across the bucket's active directories.
|
||||
pub(crate) fn verified_work(work: Vec<Self>, info: &DataUsageCacheInfo) -> Vec<Self> {
|
||||
let mut shared = HashMap::new();
|
||||
work.into_iter()
|
||||
.filter(|receipt| receipt.is_bound_to(info))
|
||||
.map(|mut receipt| {
|
||||
for page in &mut receipt.pages {
|
||||
let canonical = shared.entry(*page.digest()).or_insert_with(|| page.clone());
|
||||
*page = canonical.clone();
|
||||
}
|
||||
receipt
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
pub(crate) fn parent(&self) -> &str {
|
||||
&self.parent
|
||||
}
|
||||
|
||||
pub(super) fn manifest(&self) -> [u8; 32] {
|
||||
let mut hash = Sha256::new();
|
||||
for p in &self.pages {
|
||||
hash.update(p.digest());
|
||||
}
|
||||
hash.finalize().into()
|
||||
}
|
||||
fn checksum(&self) -> Option<[u8; 32]> {
|
||||
// Nested usage distributions can contain maps. Canonicalize those
|
||||
// map keys before hashing so restart hash randomization is irrelevant.
|
||||
let mut aggregate = serde_json::to_value(&self.aggregate).ok()?;
|
||||
aggregate.sort_all_objects();
|
||||
let header = rmp_serde::to_vec_named(&(
|
||||
self.version,
|
||||
&self.parent,
|
||||
&self.scope,
|
||||
&self.physical,
|
||||
&self.anchor,
|
||||
self.entries_read,
|
||||
self.sealed,
|
||||
&self.part,
|
||||
self.metadata,
|
||||
self.manifest(),
|
||||
aggregate,
|
||||
&self.endpoint,
|
||||
))
|
||||
.ok()?;
|
||||
Some(Sha256::digest(header).into())
|
||||
}
|
||||
pub(crate) fn is_bound_to(&self, info: &DataUsageCacheInfo) -> bool {
|
||||
self.version == 1
|
||||
&& self.scope.bucket == info.name
|
||||
&& self.scope.identity.is_valid()
|
||||
&& Some(self.scope.identity) == info.scan_identity
|
||||
&& Some(self.scope.source) == info.source
|
||||
&& info
|
||||
.scan_progress
|
||||
.is_some_and(|p| p.started_plan == self.scope.progress.started_plan)
|
||||
&& (self.parent == info.name
|
||||
|| self
|
||||
.parent
|
||||
.strip_prefix(&info.name)
|
||||
.and_then(|p| p.strip_prefix('/'))
|
||||
.is_some_and(|tail| tail.split('/').all(super::valid_component)))
|
||||
&& self.valid_payload()
|
||||
}
|
||||
pub(crate) fn read_progress(&self, info: &DataUsageCacheInfo) -> Option<DirectoryReadProgress> {
|
||||
let physical = self.physical.as_ref()?;
|
||||
(self.is_bound_to(info) && (self.sealed || self.anchor.is_some()) && self.entries_read > 0).then(|| {
|
||||
DirectoryReadProgress {
|
||||
parent: self.parent.clone(),
|
||||
physical: physical.clone(),
|
||||
position: (self.entries_read, self.sealed),
|
||||
}
|
||||
})
|
||||
}
|
||||
pub(crate) fn progress(&self, info: &DataUsageCacheInfo) -> Option<DirectoryProgress> {
|
||||
let done = self.aggregate.as_ref()?;
|
||||
let physical = self.physical.as_ref()?;
|
||||
(self.sealed && self.is_bound_to(info)).then(|| DirectoryProgress {
|
||||
parent: self.parent.clone(),
|
||||
physical: physical.clone(),
|
||||
manifest: self.manifest(),
|
||||
through: done.through.clone(),
|
||||
})
|
||||
}
|
||||
pub(crate) fn endpoint(&self) -> &str {
|
||||
&self.endpoint
|
||||
}
|
||||
fn valid_for(&self, parent: &str, owner: &Scope) -> bool {
|
||||
self.version == 1 && self.parent == parent && self.scope.may_adopt(owner) && self.valid_payload()
|
||||
}
|
||||
fn valid_payload(&self) -> bool {
|
||||
self.physical.as_ref().is_some_and(|source| source.endpoint == self.endpoint)
|
||||
&& (self.sealed || self.anchor.is_some())
|
||||
&& self.checksum() == Some(self.digest)
|
||||
&& self.pages.iter().all(NamePage::valid)
|
||||
&& self.part.as_deref().is_none_or(super::super::is_erasure_part_name)
|
||||
&& self.aggregate.as_ref().is_none_or(|done| {
|
||||
self.sealed
|
||||
&& super::valid_component(&done.through)
|
||||
&& done.value.children.is_empty()
|
||||
&& done.value.failed_objects == 0
|
||||
&& self.pages.iter().any(|page| page.contains(&done.through))
|
||||
})
|
||||
}
|
||||
pub(super) fn restore_allowed(&self, parent: &str, owner: &Scope, physical: &Option<Physical>) -> bool {
|
||||
self.valid_for(parent, owner)
|
||||
&& self.physical.is_some()
|
||||
&& &self.physical == physical
|
||||
&& (self.sealed || self.anchor.is_some())
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) type Saved = Arc<Mutex<Option<DirectoryCheckpoint>>>;
|
||||
|
||||
pub(super) struct Recorder {
|
||||
pub(super) slot: Saved,
|
||||
scope: Scope,
|
||||
parent: String,
|
||||
physical: Option<Physical>,
|
||||
pub(super) anchor: Option<native_io::DirectoryAnchor>,
|
||||
pub(super) entries_read: u64,
|
||||
pub(super) part: Option<String>,
|
||||
pub(super) metadata: bool,
|
||||
pub(super) valid_names: bool,
|
||||
endpoint: String,
|
||||
pub(super) last_update: Instant,
|
||||
}
|
||||
|
||||
impl Recorder {
|
||||
pub(super) fn new(
|
||||
slot: Saved,
|
||||
parent: &str,
|
||||
scope: Scope,
|
||||
endpoint: String,
|
||||
physical: Option<Physical>,
|
||||
previous: Option<&DirectoryCheckpoint>,
|
||||
) -> Self {
|
||||
Self {
|
||||
slot,
|
||||
scope,
|
||||
parent: parent.to_owned(),
|
||||
physical,
|
||||
endpoint,
|
||||
anchor: previous.and_then(|p| p.anchor.clone()),
|
||||
entries_read: previous.map_or(0, |p| p.entries_read),
|
||||
part: previous.and_then(|p| p.part.clone()),
|
||||
metadata: previous.is_some_and(|p| p.metadata),
|
||||
valid_names: true,
|
||||
last_update: Instant::now(),
|
||||
}
|
||||
}
|
||||
pub(super) fn save(&self, work: &DirectoryWork) {
|
||||
// An ordinary object directory is cheaper to reopen. Only a reader
|
||||
// which can actually resume or a closed directory can be checkpointed.
|
||||
let sealed = work.names.ready();
|
||||
let aggregate = if work.compacted && !work.gap {
|
||||
work.names.through().map(|through| Aggregate {
|
||||
through: through.to_owned(),
|
||||
value: work.compacted_total.clone(),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let snapshot = if self.valid_names
|
||||
&& self.physical.as_ref().is_some_and(|source| source.endpoint == self.endpoint)
|
||||
&& (sealed || self.anchor.is_some())
|
||||
{
|
||||
let mut checkpoint = DirectoryCheckpoint {
|
||||
version: 1,
|
||||
parent: self.parent.clone(),
|
||||
scope: self.scope.clone(),
|
||||
physical: self.physical.clone(),
|
||||
anchor: (!sealed).then(|| self.anchor.clone()).flatten(),
|
||||
entries_read: self.entries_read,
|
||||
pages: work.names.snapshot(),
|
||||
sealed,
|
||||
part: self.part.clone(),
|
||||
metadata: self.metadata,
|
||||
aggregate,
|
||||
endpoint: self.endpoint.clone(),
|
||||
digest: [0; 32],
|
||||
};
|
||||
checkpoint.checksum().map(|digest| {
|
||||
checkpoint.digest = digest;
|
||||
checkpoint
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
*self.slot.lock().unwrap_or_else(|p| p.into_inner()) = snapshot;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_fixture;
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl DirectoryCheckpoint {
|
||||
pub(crate) async fn wide_storage_fixture(info: &DataUsageCacheInfo) -> [Self; 2] {
|
||||
let scope = Scope {
|
||||
bucket: info.name.clone(),
|
||||
source: info.source.expect("fixture source"),
|
||||
identity: info.scan_identity.expect("fixture identity"),
|
||||
progress: info.scan_progress.expect("fixture coverage plan"),
|
||||
mode: 0,
|
||||
prefixes: None,
|
||||
watch: None,
|
||||
};
|
||||
let slot = Saved::default();
|
||||
let endpoint = info.scan_directory_source.clone().expect("fixture physical source");
|
||||
let physical = Physical {
|
||||
endpoint: endpoint.clone(),
|
||||
device: 1,
|
||||
inode: 2,
|
||||
changed: (123, 0),
|
||||
modified: SystemTime::UNIX_EPOCH,
|
||||
};
|
||||
let mut recorder = Recorder::new(slot.clone(), &info.name, scope, endpoint, Some(physical), None);
|
||||
recorder.entries_read = 300_000;
|
||||
let mut work = DirectoryWork {
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
};
|
||||
for i in 0..300_000 {
|
||||
work.names.add(format!("obj-{i:06}"));
|
||||
work.names.flush_full().await.expect("record real immutable name page");
|
||||
}
|
||||
work.names.finish();
|
||||
[121_034, 121_035].map(|objects| {
|
||||
work.names.seek_after(Some(&format!("obj-{:06}", objects - 1)));
|
||||
work.compacted_total.objects = objects;
|
||||
work.compacted_total.size = objects * 1024;
|
||||
recorder.save(&work);
|
||||
slot.lock()
|
||||
.expect("fixture slot")
|
||||
.take()
|
||||
.expect("sealed directory checkpoint")
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::BinaryHeap;
|
||||
use std::io;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const PAGE_NAMES: usize = 256;
|
||||
const PAGE_NAME_BYTES: usize = 32 * 1024;
|
||||
|
||||
/// One contiguous UTF-8 string per run. Names from the filesystem cannot
|
||||
/// contain NUL. Checkpoint clones share both the payload and its local index.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub(super) struct NamePage(Arc<PackedNames>);
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct PackedNames {
|
||||
text: Box<str>,
|
||||
digest: [u8; 32],
|
||||
#[serde(skip)]
|
||||
starts: OnceLock<Box<[usize]>>,
|
||||
#[serde(skip)]
|
||||
valid: OnceLock<bool>,
|
||||
}
|
||||
|
||||
impl NamePage {
|
||||
fn hash(text: &str) -> [u8; 32] {
|
||||
Sha256::new()
|
||||
.chain_update(b"rustfs-scanner-directory-names-v1")
|
||||
.chain_update(text.as_bytes())
|
||||
.finalize()
|
||||
.into()
|
||||
}
|
||||
|
||||
fn new(mut names: Vec<String>) -> Self {
|
||||
names.sort_unstable();
|
||||
let mut text = String::with_capacity(names.iter().map(|n| n.len() + 1).sum());
|
||||
for name in names {
|
||||
text.push_str(&name);
|
||||
text.push('\0');
|
||||
}
|
||||
Self(Arc::new(PackedNames {
|
||||
digest: Self::hash(&text),
|
||||
text: text.into_boxed_str(),
|
||||
starts: OnceLock::new(),
|
||||
valid: OnceLock::new(),
|
||||
}))
|
||||
}
|
||||
|
||||
fn starts(&self) -> &[usize] {
|
||||
self.0.starts.get_or_init(|| {
|
||||
self.names()
|
||||
.scan(0, |offset, name| {
|
||||
let start = *offset;
|
||||
*offset += name.len() + 1;
|
||||
Some(start)
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
}
|
||||
|
||||
fn at(&self, offset: usize) -> &str {
|
||||
self.0.text[offset..].split('\0').next().unwrap_or_default()
|
||||
}
|
||||
|
||||
fn get(&self, index: usize) -> Option<&str> {
|
||||
self.starts().get(index).map(|&offset| self.at(offset))
|
||||
}
|
||||
|
||||
fn after(&self, last: &str) -> usize {
|
||||
self.starts().partition_point(|&offset| self.at(offset) <= last)
|
||||
}
|
||||
|
||||
pub(super) fn contains(&self, name: &str) -> bool {
|
||||
self.starts().binary_search_by(|&offset| self.at(offset).cmp(name)).is_ok()
|
||||
}
|
||||
|
||||
pub(super) fn digest(&self) -> &[u8; 32] {
|
||||
&self.0.digest
|
||||
}
|
||||
|
||||
pub(super) fn names(&self) -> impl Iterator<Item = &str> {
|
||||
self.0.text.split_terminator('\0')
|
||||
}
|
||||
|
||||
pub(super) fn valid(&self) -> bool {
|
||||
*self.0.valid.get_or_init(|| {
|
||||
let text = &self.0.text;
|
||||
text.ends_with('\0')
|
||||
&& text.len() <= PAGE_NAME_BYTES + PAGE_NAMES + 4096
|
||||
&& self.names().take(PAGE_NAMES + 1).count() <= PAGE_NAMES
|
||||
&& self.names().all(super::valid_component)
|
||||
&& self.names().is_sorted()
|
||||
&& self.0.digest == Self::hash(text)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Immutable sorted runs are shared with checkpoint snapshots. The merge
|
||||
/// retains one head per run. At a cold resume each run seeks to the first
|
||||
/// name after the authenticated completed child using binary search.
|
||||
#[derive(Default)]
|
||||
pub(super) struct NamePages {
|
||||
pages: Vec<NamePage>,
|
||||
building: Vec<String>,
|
||||
name_bytes: usize,
|
||||
count: usize,
|
||||
heads: BinaryHeap<Reverse<(String, usize, usize)>>,
|
||||
ready: bool,
|
||||
pending: Option<String>,
|
||||
through: Option<String>,
|
||||
}
|
||||
|
||||
impl NamePages {
|
||||
pub(super) fn len(&self) -> usize {
|
||||
self.count
|
||||
}
|
||||
pub(super) fn ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
pub(super) fn through(&self) -> Option<&str> {
|
||||
self.through.as_deref()
|
||||
}
|
||||
|
||||
pub(super) fn add(&mut self, name: String) {
|
||||
self.name_bytes = self.name_bytes.saturating_add(name.len());
|
||||
self.count = self.count.saturating_add(1);
|
||||
self.building.push(name);
|
||||
}
|
||||
|
||||
fn flush(&mut self) {
|
||||
if self.building.is_empty() {
|
||||
return;
|
||||
}
|
||||
let page = NamePage::new(std::mem::take(&mut self.building));
|
||||
if let Some(first) = page.get(0) {
|
||||
self.heads.push(Reverse((first.to_owned(), self.pages.len(), 0)));
|
||||
}
|
||||
self.pages.push(page);
|
||||
self.name_bytes = 0;
|
||||
}
|
||||
|
||||
pub(super) async fn flush_full(&mut self) -> io::Result<()> {
|
||||
if self.building.len() >= PAGE_NAMES || self.name_bytes >= PAGE_NAME_BYTES {
|
||||
self.flush();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn finish(&mut self) {
|
||||
self.flush();
|
||||
self.ready = true;
|
||||
}
|
||||
|
||||
pub(super) fn snapshot(&self) -> Vec<NamePage> {
|
||||
let mut pages = self.pages.clone();
|
||||
if !self.building.is_empty() {
|
||||
pages.push(NamePage::new(self.building.clone()));
|
||||
}
|
||||
pages
|
||||
}
|
||||
|
||||
pub(super) fn restore(pages: &[NamePage], ready: bool, through: Option<&str>) -> Self {
|
||||
let mut names = Self {
|
||||
pages: pages.to_vec(),
|
||||
count: pages.iter().map(|p| p.starts().len()).sum(),
|
||||
ready,
|
||||
..Self::default()
|
||||
};
|
||||
if !ready
|
||||
&& names
|
||||
.pages
|
||||
.last()
|
||||
.is_some_and(|page| page.starts().len() < PAGE_NAMES && page.0.text.len() < PAGE_NAME_BYTES)
|
||||
&& let Some(page) = names.pages.pop()
|
||||
{
|
||||
names.building = page.names().map(str::to_owned).collect();
|
||||
names.name_bytes = names.building.iter().map(String::len).sum();
|
||||
}
|
||||
names.seek_after(through);
|
||||
names
|
||||
}
|
||||
|
||||
pub(super) fn seek_after(&mut self, through: Option<&str>) {
|
||||
self.pending = None;
|
||||
self.through = through.map(str::to_owned);
|
||||
self.heads.clear();
|
||||
for (page_index, page) in self.pages.iter().enumerate() {
|
||||
let start = through.map_or(0, |last| page.after(last));
|
||||
if let Some(next) = page.get(start) {
|
||||
self.heads.push(Reverse((next.to_owned(), page_index, start)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn rewind(&mut self) {
|
||||
self.seek_after(None);
|
||||
}
|
||||
|
||||
pub(super) async fn next(&mut self) -> io::Result<Option<String>> {
|
||||
if self.pending.is_some() {
|
||||
return Ok(self.pending.clone());
|
||||
}
|
||||
while let Some(Reverse((name, page, index))) = self.heads.pop() {
|
||||
if let Some(next) = self.pages[page].get(index + 1) {
|
||||
self.heads.push(Reverse((next.to_owned(), page, index + 1)));
|
||||
}
|
||||
if self.through.as_ref() == Some(&name) {
|
||||
continue;
|
||||
}
|
||||
self.pending = Some(name.clone());
|
||||
return Ok(Some(name));
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub(super) fn acknowledge(&mut self) -> bool {
|
||||
if let Some(name) = self.pending.take() {
|
||||
self.through = Some(name);
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn packed_membership_shares_names_and_rejects_corrupt_payloads() {
|
||||
let page = NamePage::new((0..PAGE_NAMES).rev().map(|i| format!("obj-{i:06}")).collect());
|
||||
assert!(page.valid());
|
||||
let shared = page.clone();
|
||||
assert!(Arc::ptr_eq(&page.0, &shared.0));
|
||||
let encoded = rmp_serde::to_vec_named(&page).expect("serialize packed names");
|
||||
assert!(encoded.len() < PAGE_NAMES * 12, "checkpoint stores one delimited string per page");
|
||||
let restored: NamePage = rmp_serde::from_slice(&encoded).expect("cold name page");
|
||||
assert!(restored.valid());
|
||||
assert!(restored.contains("obj-000137"));
|
||||
assert_eq!(restored.get(restored.after("obj-000137")), Some("obj-000138"));
|
||||
for text in ["", "obj", "a\0\0b\0", "b\0a\0", "../other\0"] {
|
||||
let malformed = NamePage(Arc::new(PackedNames {
|
||||
text: text.into(),
|
||||
digest: NamePage::hash(text),
|
||||
starts: OnceLock::new(),
|
||||
valid: OnceLock::new(),
|
||||
}));
|
||||
assert!(!malformed.valid(), "invalid membership accepted: {text:?}");
|
||||
}
|
||||
let mut bytes = encoded;
|
||||
let offset = bytes
|
||||
.windows(10)
|
||||
.position(|value| value == b"obj-000137")
|
||||
.expect("encoded name");
|
||||
bytes[offset] = b'x';
|
||||
assert!(!rmp_serde::from_slice::<NamePage>(&bytes).expect("decodable payload").valid());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn interleaved_pages_resume_at_each_unacknowledged_name() {
|
||||
let mut pages = NamePages::default();
|
||||
let mut expected = std::collections::BTreeSet::new();
|
||||
// Each consecutive sorted run interleaves with every other run. One
|
||||
// duplicate straddles two runs and a partial run remains in memory.
|
||||
for run in 0..5 {
|
||||
for index in (0..PAGE_NAMES).rev() {
|
||||
let name = format!("item-{index:04}-{run:02}");
|
||||
expected.insert(name.clone());
|
||||
pages.add(name);
|
||||
pages.flush_full().await.expect("append sorted run");
|
||||
}
|
||||
}
|
||||
pages.add("item-0000-02".to_owned());
|
||||
pages.finish();
|
||||
for name in &expected {
|
||||
assert_eq!(pages.next().await.expect("scheduled child").as_ref(), Some(name));
|
||||
assert_eq!(pages.next().await.expect("retry after cancellation").as_ref(), Some(name));
|
||||
assert!(pages.acknowledge());
|
||||
}
|
||||
assert_eq!(pages.next().await.expect("sealed EOF"), None);
|
||||
pages.rewind();
|
||||
assert_eq!(pages.next().await.expect("fresh coverage").as_ref(), expected.first());
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ use std::ffi::OsString;
|
||||
use std::fs::{FileType, Metadata};
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::io;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll, ready};
|
||||
@@ -18,31 +18,156 @@ use crate::storage_api::ecstore_spawn_blocking_with_resource_ownership as spawn_
|
||||
// Keep Tokio's existing bounded prefetch and one native job per refill.
|
||||
const DIRECTORY_CHUNK_SIZE: usize = 32;
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
mod positioned;
|
||||
|
||||
/// Identifies a verified prefix of a physical directory buffer. Only the
|
||||
/// matching directory generation can use its platform-specific kernel cookie.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct DirectoryAnchor {
|
||||
directory: [u8; 32],
|
||||
cookie: u64,
|
||||
entries: u32,
|
||||
prefix: [u8; 32],
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct DirEntry {
|
||||
entry: Arc<std::fs::DirEntry>,
|
||||
source: EntrySource,
|
||||
file_type: Option<FileType>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
enum EntrySource {
|
||||
Native(Arc<std::fs::DirEntry>),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
Positioned(Arc<PositionedEntry>),
|
||||
Recorded {
|
||||
parent: Arc<std::path::Path>,
|
||||
name: OsString,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[derive(Clone)]
|
||||
struct PositionedEntry {
|
||||
parent: Arc<Path>,
|
||||
name: OsString,
|
||||
anchor: Option<DirectoryAnchor>,
|
||||
}
|
||||
|
||||
impl DirEntry {
|
||||
pub(super) fn recorded(parent: Arc<std::path::Path>, name: OsString, file_type: Option<FileType>) -> Self {
|
||||
Self {
|
||||
source: EntrySource::Recorded { parent, name },
|
||||
file_type,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn anchor(&self) -> Option<&DirectoryAnchor> {
|
||||
match &self.source {
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
EntrySource::Positioned(entry) => entry.anchor.as_ref(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn without_anchor(self) -> Self {
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
{
|
||||
let mut entry = self;
|
||||
if let EntrySource::Positioned(position) = &mut entry.source {
|
||||
Arc::make_mut(position).anchor = None;
|
||||
}
|
||||
entry
|
||||
}
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn is_recorded(&self) -> bool {
|
||||
matches!(&self.source, EntrySource::Recorded { .. })
|
||||
}
|
||||
|
||||
pub(super) fn cached_type(&self) -> Option<FileType> {
|
||||
self.file_type
|
||||
}
|
||||
|
||||
pub(super) fn path(&self) -> PathBuf {
|
||||
self.entry.path()
|
||||
match &self.source {
|
||||
EntrySource::Native(entry) => entry.path(),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
EntrySource::Positioned(entry) => entry.parent.join(&entry.name),
|
||||
EntrySource::Recorded { parent, name } => parent.join(name),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn file_name(&self) -> OsString {
|
||||
self.entry.file_name()
|
||||
match &self.source {
|
||||
EntrySource::Native(entry) => entry.file_name(),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
EntrySource::Positioned(entry) => entry.name.clone(),
|
||||
EntrySource::Recorded { name, .. } => name.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn file_type(&self) -> io::Result<FileType> {
|
||||
if let Some(file_type) = self.file_type {
|
||||
return Ok(file_type);
|
||||
}
|
||||
let entry = self.entry.clone();
|
||||
spawn_native(move || entry.file_type()).await.map_err(io::Error::other)?
|
||||
match &self.source {
|
||||
EntrySource::Native(entry) => {
|
||||
let entry = entry.clone();
|
||||
spawn_native(move || entry.file_type()).await.map_err(io::Error::other)?
|
||||
}
|
||||
_ => {
|
||||
let path = self.path();
|
||||
spawn_native(move || std::fs::symlink_metadata(path).map(|meta| meta.file_type()))
|
||||
.await
|
||||
.map_err(io::Error::other)?
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PhysicalReader {
|
||||
Standard(std::fs::ReadDir),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
Positioned(Box<positioned::Reader>),
|
||||
}
|
||||
|
||||
impl PhysicalReader {
|
||||
fn open(path: &Path, indexed: bool, anchor: Option<&DirectoryAnchor>) -> io::Result<Self> {
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
if indexed {
|
||||
return positioned::Reader::open(path, anchor).map(Box::new).map(Self::Positioned);
|
||||
}
|
||||
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
|
||||
let _ = indexed;
|
||||
if anchor.is_some() {
|
||||
return Err(io::ErrorKind::Unsupported.into());
|
||||
}
|
||||
std::fs::read_dir(path).map(Self::Standard)
|
||||
}
|
||||
|
||||
fn next(&mut self) -> Option<io::Result<DirEntry>> {
|
||||
match self {
|
||||
Self::Standard(reader) => reader.next().map(|entry| {
|
||||
entry.map(|entry| DirEntry {
|
||||
file_type: entry.file_type().ok(),
|
||||
source: EntrySource::Native(Arc::new(entry)),
|
||||
})
|
||||
}),
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
Self::Positioned(reader) => reader.next(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct DirectoryBuffer {
|
||||
reader: std::fs::ReadDir,
|
||||
reader: PhysicalReader,
|
||||
entries: VecDeque<io::Result<DirEntry>>,
|
||||
more: bool,
|
||||
}
|
||||
@@ -55,10 +180,7 @@ impl DirectoryBuffer {
|
||||
break;
|
||||
};
|
||||
let success = entry.is_ok();
|
||||
self.entries.push_back(entry.map(|entry| DirEntry {
|
||||
file_type: entry.file_type().ok(),
|
||||
entry: Arc::new(entry),
|
||||
}));
|
||||
self.entries.push_back(entry);
|
||||
if !success {
|
||||
break;
|
||||
}
|
||||
@@ -74,14 +196,26 @@ enum DirectoryState {
|
||||
|
||||
pub(super) struct ReadDir {
|
||||
state: DirectoryState,
|
||||
#[cfg(test)]
|
||||
slow_directory: bool,
|
||||
}
|
||||
|
||||
impl ReadDir {
|
||||
pub(super) async fn open(path: &str) -> io::Result<Self> {
|
||||
pub(super) async fn open(path: &str, indexed: bool) -> io::Result<Self> {
|
||||
Self::open_directory(path, indexed, None).await
|
||||
}
|
||||
|
||||
pub(super) async fn open_at(path: &str, anchor: &DirectoryAnchor) -> io::Result<Self> {
|
||||
Self::open_directory(path, true, Some(anchor.clone())).await
|
||||
}
|
||||
|
||||
async fn open_directory(path: &str, indexed: bool, anchor: Option<DirectoryAnchor>) -> io::Result<Self> {
|
||||
#[cfg(test)]
|
||||
let slow_directory = super::tests::fair_prefix_quantum::is_slow_directory(path);
|
||||
let path = PathBuf::from(path);
|
||||
let buffer = spawn_native(move || {
|
||||
let mut buffer = DirectoryBuffer {
|
||||
reader: std::fs::read_dir(path)?,
|
||||
reader: PhysicalReader::open(&path, indexed, anchor.as_ref())?,
|
||||
entries: VecDeque::with_capacity(DIRECTORY_CHUNK_SIZE),
|
||||
more: true,
|
||||
};
|
||||
@@ -92,12 +226,18 @@ impl ReadDir {
|
||||
.map_err(io::Error::other)??;
|
||||
Ok(Self {
|
||||
state: DirectoryState::Idle(Some(buffer)),
|
||||
#[cfg(test)]
|
||||
slow_directory,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) async fn next_entry(&mut self) -> io::Result<Option<DirEntry>> {
|
||||
// Keep a pending refill on the reader, not in the borrowed next-entry
|
||||
// future. Cancelling that future must not skip its prefetched entries.
|
||||
#[cfg(test)]
|
||||
if self.slow_directory {
|
||||
super::tests::fair_prefix_quantum::charge_physical_read().await;
|
||||
}
|
||||
poll_fn(|cx| self.poll_next_entry(cx)).await
|
||||
}
|
||||
|
||||
@@ -201,7 +341,7 @@ mod tests {
|
||||
Resources::Layered(owner.capture(&units.iter().collect::<Vec<_>>()).expect("native ownership"))
|
||||
};
|
||||
drop(units);
|
||||
let mut reader = ReadDir::open(path).await.expect("prepare reader");
|
||||
let mut reader = ReadDir::open(path, false).await.expect("prepare reader");
|
||||
let mut entry = reader.next_entry().await.expect("first entry").expect("file");
|
||||
entry.file_type = None;
|
||||
for _ in 1..DIRECTORY_CHUNK_SIZE {
|
||||
@@ -210,7 +350,7 @@ mod tests {
|
||||
let (release, blocker) = occupy_native_worker().await;
|
||||
let work = async {
|
||||
match operation {
|
||||
"open" => ReadDir::open(path).await.map(|_| ()),
|
||||
"open" => ReadDir::open(path, false).await.map(|_| ()),
|
||||
"refill" => reader.next_entry().await.map(|_| ()),
|
||||
"metadata" => metadata(entry.path().to_str().expect("entry path")).await.map(|_| ()),
|
||||
"file_type" => entry.file_type().await.map(|_| ()),
|
||||
@@ -253,7 +393,9 @@ mod tests {
|
||||
runtime().block_on(async {
|
||||
let count = DIRECTORY_CHUNK_SIZE * 3 + 7;
|
||||
let root = directory(count);
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path")).await.expect("open");
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path"), false)
|
||||
.await
|
||||
.expect("open");
|
||||
let mut names = std::collections::BTreeSet::new();
|
||||
for _ in 0..DIRECTORY_CHUNK_SIZE {
|
||||
let entry = reader.next_entry().await.expect("entry").expect("file");
|
||||
@@ -273,12 +415,71 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[test]
|
||||
fn reopened_directory_matches_original_suffix_and_rejects_stale_or_corrupt_positions() {
|
||||
runtime().block_on(async {
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
let root = directory(900);
|
||||
let path = root.path().to_str().expect("root path");
|
||||
#[cfg(target_os = "linux")]
|
||||
let raw_name = std::ffi::OsStr::from_bytes(b"object-\xff");
|
||||
// APFS requires filenames that are valid Unicode.
|
||||
#[cfg(target_os = "macos")]
|
||||
let raw_name = std::ffi::OsStr::new("object-目录");
|
||||
std::fs::write(root.path().join(raw_name), []).expect("platform-native directory entry");
|
||||
let mut reader = ReadDir::open(path, true).await.expect("open indexed directory");
|
||||
let mut names = Vec::new();
|
||||
let mut anchors = Vec::new();
|
||||
while let Some(entry) = reader.next_entry().await.expect("read indexed entry") {
|
||||
assert!(!entry.is_recorded());
|
||||
names.push(entry.file_name());
|
||||
anchors.push(entry.anchor().expect("native directory position").clone());
|
||||
}
|
||||
drop(reader);
|
||||
assert!(names.iter().any(|name| name == raw_name));
|
||||
assert!(anchors.iter().any(|anchor| anchor.cookie != 0), "exercise a later kernel buffer");
|
||||
for i in [0, 32, 145, 419, names.len() - 1] {
|
||||
let mut resumed = ReadDir::open_at(path, &anchors[i])
|
||||
.await
|
||||
.expect("reopen directory at verified entry");
|
||||
for expected in &names[i + 1..] {
|
||||
assert_eq!(
|
||||
&resumed
|
||||
.next_entry()
|
||||
.await
|
||||
.expect("suffix read")
|
||||
.expect("suffix entry")
|
||||
.file_name(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
assert!(resumed.next_entry().await.expect("resumed EOF").is_none());
|
||||
}
|
||||
|
||||
let mut corrupt = anchors[0].clone();
|
||||
corrupt.prefix[0] ^= 1;
|
||||
assert!(matches!(ReadDir::open_at(path, &corrupt).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
|
||||
corrupt.entries = u32::MAX;
|
||||
assert!(matches!(ReadDir::open_at(path, &corrupt).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
|
||||
let other = directory(1);
|
||||
let another = other.path().to_str().expect("other directory");
|
||||
assert!(matches!(ReadDir::open_at(another, &anchors[0]).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
|
||||
// Removing the bound entry also invalidates its directory generation.
|
||||
std::fs::remove_file(root.path().join(&names[0])).expect("remove anchor entry");
|
||||
assert!(matches!(ReadDir::open_at(path, &anchors[0]).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_prefetch_preserves_empty_and_exact_chunk_boundaries() {
|
||||
runtime().block_on(async {
|
||||
for count in [0, 1, DIRECTORY_CHUNK_SIZE, DIRECTORY_CHUNK_SIZE + 1] {
|
||||
let root = directory(count);
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path")).await.expect("open");
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path"), false)
|
||||
.await
|
||||
.expect("open");
|
||||
let mut names = std::collections::BTreeSet::new();
|
||||
while let Some(entry) = reader.next_entry().await.expect("read entry") {
|
||||
assert!(names.insert(entry.file_name()), "prefetch duplicated an entry");
|
||||
@@ -293,7 +494,9 @@ mod tests {
|
||||
fn failed_refill_is_an_error_after_partial_enumeration() {
|
||||
runtime().block_on(async {
|
||||
let root = directory(DIRECTORY_CHUNK_SIZE + 1);
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path")).await.expect("open");
|
||||
let mut reader = ReadDir::open(root.path().to_str().expect("root path"), false)
|
||||
.await
|
||||
.expect("open");
|
||||
for _ in 0..DIRECTORY_CHUNK_SIZE {
|
||||
assert!(reader.next_entry().await.expect("prefetched entry").is_some());
|
||||
}
|
||||
@@ -322,10 +525,10 @@ mod tests {
|
||||
let root = directory(1);
|
||||
let path = root.path().join("missing");
|
||||
let missing = path.to_str().expect("missing path");
|
||||
assert!(matches!(ReadDir::open(missing).await, Err(error) if error.kind() == io::ErrorKind::NotFound));
|
||||
assert!(matches!(ReadDir::open(missing, false).await, Err(error) if error.kind() == io::ErrorKind::NotFound));
|
||||
assert!(matches!(metadata(missing).await, Err(error) if error.kind() == io::ErrorKind::NotFound));
|
||||
let file = root.path().join("object-0");
|
||||
assert!(ReadDir::open(file.to_str().expect("file path")).await.is_err());
|
||||
assert!(ReadDir::open(file.to_str().expect("file path"), false).await.is_err());
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let link = root.path().join("link");
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Reopen an indexed directory at the last entry committed by the walker.
|
||||
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::{File, FileType, Metadata};
|
||||
use std::io;
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustix::fs::{Dir, FileType as NativeType, Mode, OFlags, SeekFrom};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{DirEntry, DirectoryAnchor, EntrySource, PositionedEntry};
|
||||
|
||||
// Darwin's dirent does not expose Linux d_off. Use the kernel buffer's start
|
||||
// position and verify the consumed prefix of that buffer on a fresh handle.
|
||||
#[cfg(target_os = "macos")]
|
||||
const MAX_ANCHOR_PREFIX: u32 = 4096;
|
||||
#[cfg(target_os = "linux")]
|
||||
const MAX_ANCHOR_PREFIX: u32 = 1;
|
||||
|
||||
fn changed() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "scanner directory position no longer matches its checkpoint")
|
||||
}
|
||||
|
||||
fn resume_error(error: rustix::io::Errno) -> io::Error {
|
||||
use rustix::io::Errno;
|
||||
match error {
|
||||
Errno::SPIPE | Errno::OPNOTSUPP => io::ErrorKind::Unsupported.into(),
|
||||
Errno::INVAL | Errno::OVERFLOW | Errno::STALE => changed(),
|
||||
other => other.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn hash_record(prefix: &mut Sha256, record: &rustix::fs::DirEntry) {
|
||||
prefix.update(record.ino().to_le_bytes());
|
||||
// Names cannot contain NUL. Include the terminator to delimit records.
|
||||
prefix.update(record.file_name().to_bytes_with_nul());
|
||||
}
|
||||
|
||||
fn identity(meta: &Metadata) -> [u8; 32] {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(std::env::consts::OS.as_bytes());
|
||||
for value in [
|
||||
meta.dev().to_le_bytes(),
|
||||
meta.ino().to_le_bytes(),
|
||||
meta.ctime().to_le_bytes(),
|
||||
meta.ctime_nsec().to_le_bytes(),
|
||||
meta.mtime().to_le_bytes(),
|
||||
meta.mtime_nsec().to_le_bytes(),
|
||||
] {
|
||||
digest.update(value);
|
||||
}
|
||||
digest.finalize().into()
|
||||
}
|
||||
|
||||
pub(super) struct Reader {
|
||||
dir: Dir,
|
||||
parent: Arc<Path>,
|
||||
directory_type: FileType,
|
||||
regular_type: Option<FileType>,
|
||||
link_type: Option<FileType>,
|
||||
identity: [u8; 32],
|
||||
next_cookie: Option<u64>,
|
||||
#[cfg(target_os = "macos")]
|
||||
page: DarwinPage,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
struct DarwinPage {
|
||||
position: File,
|
||||
start: Option<u64>,
|
||||
count: u32,
|
||||
prefix: Sha256,
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
pub(super) fn open(path: &Path, resume: Option<&DirectoryAnchor>) -> io::Result<Self> {
|
||||
let fd = File::from(rustix::fs::open(
|
||||
path,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
|
||||
Mode::empty(),
|
||||
)?);
|
||||
let meta = fd.metadata()?;
|
||||
let identity = identity(&meta);
|
||||
let cookie = resume.map_or(0, |anchor| anchor.cookie);
|
||||
if let Some(anchor) = resume {
|
||||
if anchor.directory != identity || !(1..=MAX_ANCHOR_PREFIX).contains(&anchor.entries) {
|
||||
return Err(changed());
|
||||
}
|
||||
if rustix::fs::seek(&fd, SeekFrom::Start(cookie)).map_err(resume_error)? != cookie {
|
||||
return Err(changed());
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
let page = DarwinPage {
|
||||
position: fd.try_clone()?,
|
||||
start: Some(cookie),
|
||||
count: 0,
|
||||
prefix: Sha256::new(),
|
||||
};
|
||||
let mut reader = Self {
|
||||
dir: Dir::new(fd)?,
|
||||
parent: Arc::from(path),
|
||||
directory_type: meta.file_type(),
|
||||
regular_type: None,
|
||||
link_type: None,
|
||||
identity,
|
||||
next_cookie: Some(cookie),
|
||||
#[cfg(target_os = "macos")]
|
||||
page,
|
||||
};
|
||||
if let Some(expected) = resume {
|
||||
// A fresh libc stream can choose a different buffer size. Validate
|
||||
// exactly the saved prefix, regardless of its new buffer boundaries.
|
||||
let mut prefix = Sha256::new();
|
||||
for _ in 0..expected.entries {
|
||||
let record = reader.dir.read().transpose().map_err(resume_error)?.ok_or_else(changed)?;
|
||||
hash_record(&mut prefix, &record);
|
||||
reader.anchor(&record);
|
||||
}
|
||||
if prefix.finalize()[..] != expected.prefix {
|
||||
return Err(changed());
|
||||
}
|
||||
}
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
fn anchor(&mut self, record: &rustix::fs::DirEntry) -> Option<DirectoryAnchor> {
|
||||
#[cfg(target_os = "linux")]
|
||||
let mut single = Sha256::new();
|
||||
#[cfg(target_os = "linux")]
|
||||
let (cookie, entries, prefix) = {
|
||||
let cookie = self.next_cookie;
|
||||
self.next_cookie = u64::try_from(record.offset()).ok();
|
||||
(cookie?, 1, &mut single)
|
||||
};
|
||||
#[cfg(target_os = "macos")]
|
||||
let (cookie, entries, prefix) = {
|
||||
let after = rustix::fs::tell(&self.page.position).ok();
|
||||
if after != self.next_cookie {
|
||||
self.page.start = self.next_cookie;
|
||||
self.next_cookie = after;
|
||||
self.page.count = 0;
|
||||
self.page.prefix = Sha256::new();
|
||||
}
|
||||
self.page.count = self.page.count.saturating_add(1);
|
||||
if self.page.count > MAX_ANCHOR_PREFIX {
|
||||
return None;
|
||||
}
|
||||
(self.page.start?, self.page.count, &mut self.page.prefix)
|
||||
};
|
||||
hash_record(prefix, record);
|
||||
Some(DirectoryAnchor {
|
||||
directory: self.identity,
|
||||
cookie,
|
||||
entries,
|
||||
prefix: prefix.clone().finalize().into(),
|
||||
})
|
||||
}
|
||||
|
||||
fn file_type(&mut self, kind: NativeType, name: &OsStr) -> Option<FileType> {
|
||||
// std::fs::FileType has no public constructor. Reuse the observed type
|
||||
// only when the kernel dirent identified the same class. DT_UNKNOWN
|
||||
// and special files retain the per-entry symlink_metadata fallback.
|
||||
let cached = match kind {
|
||||
NativeType::Directory => return Some(self.directory_type),
|
||||
NativeType::RegularFile => &mut self.regular_type,
|
||||
NativeType::Symlink => &mut self.link_type,
|
||||
_ => return None,
|
||||
};
|
||||
if cached.is_none() {
|
||||
let ty = std::fs::symlink_metadata(self.parent.join(name)).ok()?.file_type();
|
||||
if (ty.is_file() && kind == NativeType::RegularFile) || (ty.is_symlink() && kind == NativeType::Symlink) {
|
||||
*cached = Some(ty);
|
||||
}
|
||||
return Some(ty);
|
||||
}
|
||||
*cached
|
||||
}
|
||||
|
||||
pub(super) fn next(&mut self) -> Option<io::Result<DirEntry>> {
|
||||
loop {
|
||||
let raw = match self.dir.read()? {
|
||||
Ok(record) => record,
|
||||
Err(error) => return Some(Err(error.into())),
|
||||
};
|
||||
let anchor = self.anchor(&raw);
|
||||
let bytes = raw.file_name().to_bytes();
|
||||
if matches!(bytes, b"." | b"..") {
|
||||
continue;
|
||||
}
|
||||
let name = OsStr::from_bytes(bytes);
|
||||
let file_type = self.file_type(raw.file_type(), name);
|
||||
return Some(Ok(DirEntry {
|
||||
source: EntrySource::Positioned(Arc::new(PositionedEntry {
|
||||
parent: self.parent.clone(),
|
||||
name: name.into(),
|
||||
anchor,
|
||||
})),
|
||||
file_type,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -30,7 +30,7 @@ use std::sync::Mutex;
|
||||
|
||||
pub(super) mod checkpoint_fixture;
|
||||
pub(super) mod enumeration_restart;
|
||||
mod fair_prefix_quantum;
|
||||
pub(super) mod fair_prefix_quantum;
|
||||
mod incremental_enumeration;
|
||||
mod listing_completion;
|
||||
mod metadata_visits;
|
||||
@@ -403,6 +403,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
|
||||
resume_failed_paths: Vec::new(),
|
||||
coverage_gap: false,
|
||||
raw_enumeration_progress: Vec::new(),
|
||||
directory_checkpoints: HashMap::new(),
|
||||
pending_size_reconciliation_keys: HashSet::new(),
|
||||
pending_size_reconciliation_scopes: HashSet::new(),
|
||||
pending_size_reconciliation_truncated: false,
|
||||
@@ -3103,112 +3104,6 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cache_reuse_gates_report_only_the_first_production_condition() {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/child",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
compacted: true,
|
||||
objects: 2,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let hash = hash_path("bucket/child");
|
||||
let folder = CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
let child = CachedFolder {
|
||||
name: "bucket/child".to_string(),
|
||||
parent: Some(hash_path("bucket")),
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
let abandoned = HashSet::from([hash.0.clone()]);
|
||||
let before = global_metrics().scan_observation().snapshot();
|
||||
|
||||
assert!(!scanner.should_reuse_clean_root_child(&folder, &DataUsageEntry::default(), &child, &hash, &abandoned));
|
||||
let missing_scope = global_metrics().scan_observation().snapshot();
|
||||
assert_eq!(
|
||||
missing_scope.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"],
|
||||
before.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"] + 1
|
||||
);
|
||||
scanner.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".to_string()]));
|
||||
scanner.is_erasure_mode = true;
|
||||
assert!(!scanner.should_reuse_clean_root_child(&folder, &DataUsageEntry::default(), &child, &hash, &abandoned));
|
||||
let erasure = global_metrics().scan_observation().snapshot();
|
||||
assert_eq!(
|
||||
erasure.reuse_decisions["clean_prefix"].blocked["erasure_health_required"],
|
||||
before.reuse_decisions["clean_prefix"].blocked["erasure_health_required"] + 1
|
||||
);
|
||||
assert_eq!(erasure.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks + 1);
|
||||
|
||||
assert!(!scanner.can_reuse_covered_namespace("bucket/child", true));
|
||||
scanner.old_cache.info.scan_progress = Some(crate::DataUsageScanProgress {
|
||||
started_plan: crate::DataUsageScanPlanDigest([1; 32]),
|
||||
requested_plan: crate::DataUsageScanPlanDigest([1; 32]),
|
||||
});
|
||||
assert!(!scanner.can_reuse_covered_namespace("bucket/child", true));
|
||||
assert!(!scanner.can_reuse_covered_namespace("bucket/child", false));
|
||||
scanner.resume_frontier = Some("bucket/child".to_string());
|
||||
scanner.resume_failed_paths = vec!["bucket/child/failed".to_string()];
|
||||
assert!(!scanner.can_reuse_covered_namespace("bucket/child", false));
|
||||
scanner.resume_failed_paths.clear();
|
||||
assert!(scanner.can_reuse_covered_namespace("bucket/child", false));
|
||||
let covered = global_metrics().scan_observation().snapshot();
|
||||
let reasons = &covered.reuse_decisions["covered_namespace"];
|
||||
let prior = &before.reuse_decisions["covered_namespace"];
|
||||
assert_eq!(reasons.not_evaluated["no_resume_sweep"], prior.not_evaluated["no_resume_sweep"] + 1);
|
||||
assert_eq!(reasons.blocked["compacted_parent"], prior.blocked["compacted_parent"] + 1);
|
||||
assert_eq!(reasons.blocked["missing_coverage"], prior.blocked["missing_coverage"] + 1);
|
||||
assert_eq!(reasons.blocked["cached_failures"], prior.blocked["cached_failures"] + 1);
|
||||
assert_eq!(reasons.eligible, prior.eligible + 1);
|
||||
|
||||
with_var(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, Some("16"), || {
|
||||
scanner.scan_mode = HealScanMode::Deep;
|
||||
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, true), None);
|
||||
scanner.scan_mode = HealScanMode::Normal;
|
||||
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, true), None);
|
||||
let replication: ReplicationConfig = serde_json::from_value(serde_json::json!({
|
||||
"config": { "role": "", "rules": [{ "destination": { "bucket": "arn:test" }, "id": "status", "prefix": "", "priority": 1, "status": "Enabled" }] },
|
||||
"remotes": crate::storage_api::EcstoreBucketTargets::default(),
|
||||
})).expect("active replication fixture");
|
||||
scanner.old_cache.info.replication = Some(Arc::new(replication));
|
||||
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, false), None);
|
||||
scanner.old_cache.info.replication = None;
|
||||
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, false), None);
|
||||
let rescan_cycle = (0..16)
|
||||
.find(|cycle| hash.mod_(*cycle, 16))
|
||||
.expect("hash is visited in one sampling cycle");
|
||||
scanner.old_cache.info.next_cycle = u64::from(rescan_cycle);
|
||||
assert_eq!(scanner.sampled_namespace_reuse(&hash, false, false), Some(false));
|
||||
scanner.old_cache.info.next_cycle = u64::from((rescan_cycle + 1) % 16);
|
||||
assert_eq!(scanner.sampled_namespace_reuse(&hash, false, false), Some(true));
|
||||
});
|
||||
let after = global_metrics().scan_observation().snapshot();
|
||||
let reasons = &after.reuse_decisions["namespace_sampling"];
|
||||
let prior = &before.reuse_decisions["namespace_sampling"];
|
||||
for label in ["deep_scan", "forward_sweep", "active_replication"] {
|
||||
assert_eq!(
|
||||
reasons.not_evaluated[label],
|
||||
prior.not_evaluated[label] + 1,
|
||||
"first sampling condition {label}"
|
||||
);
|
||||
}
|
||||
assert_eq!(reasons.blocked["compacted_parent"], prior.blocked["compacted_parent"] + 1);
|
||||
assert_eq!(reasons.blocked["selected_for_walk"], prior.blocked["selected_for_walk"] + 1);
|
||||
assert_eq!(reasons.eligible, prior.eligible + 1);
|
||||
for (kind, reuse) in &after.cache_reuse {
|
||||
assert_eq!(reuse.count, before.cache_reuse[kind].count, "gate admission alone does not copy {kind}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entries() {
|
||||
@@ -3250,21 +3145,18 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
let before = global_metrics().scan_observation().snapshot();
|
||||
// The sorted queue can reach a budget stop after linking a clean prefix
|
||||
// but before the first dirty child advances the durable sweep frontier.
|
||||
assert!(!scanner.can_reuse_covered_namespace("bucket/clean", false));
|
||||
assert!(scanner.can_restore_linked_child(&folder, &DataUsageEntry::default(), "bucket/clean"));
|
||||
assert!(!scanner.can_restore_linked_child(&folder, &DataUsageEntry::default(), "bucket/dirty"));
|
||||
assert!(!scanner.can_restore_linked_child(&folder, &DataUsageEntry::default(), "bucket/absent"));
|
||||
let mut root = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(CancellationToken::new(), folder, &mut root)
|
||||
.await
|
||||
.expect("scoped root scan should finish successfully");
|
||||
|
||||
let observed = global_metrics().scan_observation().snapshot();
|
||||
assert_eq!(observed.cache_reuse["clean_prefix"].count, before.cache_reuse["clean_prefix"].count + 1);
|
||||
assert_eq!(observed.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks);
|
||||
assert_eq!(
|
||||
observed.reuse_decisions["clean_prefix"].eligible,
|
||||
before.reuse_decisions["clean_prefix"].eligible + 1
|
||||
);
|
||||
|
||||
let clean = scanner
|
||||
.new_cache
|
||||
.size_recursive("bucket/clean")
|
||||
@@ -3282,78 +3174,6 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
|
||||
assert_eq!((bucket.size, bucket.objects), (17, 3));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_cache_observations_distinguish_checkpoint_and_periodic_sampling() {
|
||||
for covered in [true, false] {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
tokio::fs::create_dir_all(temp_dir.join("bucket/child"))
|
||||
.await
|
||||
.expect("create cached namespace");
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/child",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 17,
|
||||
objects: 5,
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let kind = if covered {
|
||||
scanner.old_cache.info.scan_progress = Some(crate::DataUsageScanProgress {
|
||||
started_plan: crate::DataUsageScanPlanDigest([1; 32]),
|
||||
requested_plan: crate::DataUsageScanPlanDigest([1; 32]),
|
||||
});
|
||||
scanner.resume_frontier = Some("bucket/child".to_string());
|
||||
"covered_namespace"
|
||||
} else {
|
||||
let hash = hash_path("bucket/child");
|
||||
let cycle = (0..data_usage_update_dir_cycles())
|
||||
.find(|cycle| !hash.mod_(*cycle, data_usage_update_dir_cycles()))
|
||||
.expect("a cycle samples other work");
|
||||
scanner.old_cache.info.next_cycle = u64::from(cycle);
|
||||
"sampled_namespace"
|
||||
};
|
||||
let before = global_metrics().scan_observation().snapshot();
|
||||
let mut root = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(
|
||||
CancellationToken::new(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut root,
|
||||
)
|
||||
.await
|
||||
.expect("cached namespace reuses accounting");
|
||||
let summary = scanner.new_cache.size_recursive("bucket").expect("complete bucket totals");
|
||||
assert_eq!((summary.size, summary.objects), (17, 5));
|
||||
let after = global_metrics().scan_observation().snapshot();
|
||||
for label in [
|
||||
"clean_prefix",
|
||||
"covered_namespace",
|
||||
"sampled_namespace",
|
||||
"classified_namespace",
|
||||
] {
|
||||
assert_eq!(
|
||||
after.cache_reuse[label].count,
|
||||
before.cache_reuse[label].count + u64::from(label == kind),
|
||||
"reuse reason {label}"
|
||||
);
|
||||
}
|
||||
assert_eq!(after.metadata_reads.attempts, before.metadata_reads.attempts);
|
||||
let gate = if covered { "covered_namespace" } else { "namespace_sampling" };
|
||||
assert_eq!(after.reuse_decisions[gate].eligible, before.reuse_decisions[gate].eligible + 1);
|
||||
}
|
||||
}
|
||||
|
||||
async fn scan_hot_cold_segment_fixture(scoped: bool) -> (DataUsageEntry, Vec<String>) {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let visits = metadata_visits::observe(&mut scanner);
|
||||
@@ -3465,7 +3285,6 @@ async fn scoped_root_scan_preserves_erasure_health_walks() {
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
let before = global_metrics().scan_observation().snapshot();
|
||||
let mut root = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(CancellationToken::new(), folder, &mut root)
|
||||
@@ -3477,9 +3296,6 @@ async fn scoped_root_scan_preserves_erasure_health_walks() {
|
||||
.size_recursive("bucket/clean")
|
||||
.expect("erasure scan should visit the clean entry");
|
||||
assert_eq!((clean.size, clean.objects), (0, 0));
|
||||
let observed = global_metrics().scan_observation().snapshot();
|
||||
assert_eq!(observed.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks + 1);
|
||||
assert_eq!(observed.cache_reuse["clean_prefix"].count, before.cache_reuse["clean_prefix"].count);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1129,7 +1129,7 @@ async fn checkpoint_admission_log_capture_survives_another_future_dispatcher() {
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_checkpoint_object(root: &std::path::Path, object: &str, versions: &[(Option<Uuid>, i64)]) {
|
||||
pub(super) async fn write_checkpoint_object(root: &std::path::Path, object: &str, versions: &[(Option<Uuid>, i64)]) {
|
||||
let mut metadata = FileMeta::new();
|
||||
for (index, (version_id, size)) in versions.iter().enumerate() {
|
||||
let mut info = FileInfo::new(object, 4, 2);
|
||||
|
||||
@@ -820,6 +820,11 @@ async fn periodic_nested_snapshot_adoption_records_checkpoint_used_on_real_scan_
|
||||
.await
|
||||
.expect("create the real cached subtree directory");
|
||||
let (mut cache, _) = bound_checkpoint();
|
||||
cache.info.scan_directory_source = Some(
|
||||
directory_cursor::physical_source(&scanner.local_disk)
|
||||
.await
|
||||
.expect("fixture disk source"),
|
||||
);
|
||||
cache.info.skip_healing = true;
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
@@ -856,6 +861,80 @@ async fn periodic_nested_snapshot_adoption_records_checkpoint_used_on_real_scan_
|
||||
assert_eq!(budget.reason(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn periodic_compacted_checkpoint_survives_a_crash_inside_the_first_quantum() {
|
||||
use super::super::{enumeration_restart::seed_flat_bucket, fair_prefix_quantum::run_until_finished};
|
||||
let (mut scanner, root) = build_test_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
};
|
||||
// seed_flat_bucket creates its directory; retain the allocated native
|
||||
// disk and initialize this already-created bucket in the same shape.
|
||||
let template = tempfile::tempdir().expect("fixture directory");
|
||||
let cache = seed_flat_bucket(&template.path().join("initial"), 0).await;
|
||||
for index in 0..40 {
|
||||
write_checkpoint_object(&root, &format!("object-{index:04}"), &[(Some(Uuid::from_u128(index + 1)), 3)]).await;
|
||||
}
|
||||
prepare_scanner(&mut scanner, cache);
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
scanner.checkpoint_tx = Some(tx);
|
||||
scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL - 1;
|
||||
scanner.last_checkpoint_at = Instant::now()
|
||||
.checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL)
|
||||
.expect("checkpoint interval");
|
||||
let ctx = CancellationToken::new();
|
||||
scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
|
||||
let mut entry = DataUsageEntry::default();
|
||||
super::super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
let checkpoint = {
|
||||
let scan = scanner.scan_folder(
|
||||
ctx.clone(),
|
||||
CachedFolder {
|
||||
name: "bucket".to_owned(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
},
|
||||
&mut entry,
|
||||
);
|
||||
tokio::pin!(scan);
|
||||
tokio::select! {
|
||||
checkpoint=rx.recv()=>checkpoint.expect("compacted owner emitted a periodic checkpoint"),
|
||||
result=&mut scan=>panic!("scan returned before it checkpointed a completed compacted child: {result:?}"),
|
||||
}
|
||||
};
|
||||
assert!(!ctx.is_cancelled());
|
||||
assert!(!checkpoint.info.snapshot_complete);
|
||||
assert!(
|
||||
checkpoint
|
||||
.info
|
||||
.scan_directory_work
|
||||
.iter()
|
||||
.any(|receipt| receipt.progress(&checkpoint.info).is_some())
|
||||
);
|
||||
let total = checkpoint.checked_flatten("bucket").expect("connected partial root");
|
||||
assert!((1..40).contains(&total.objects));
|
||||
assert_eq!(total.size, total.objects * 3);
|
||||
let bytes = checkpoint.marshal_msg().expect("persist running checkpoint");
|
||||
drop(scanner);
|
||||
let durable = DataUsageCache::unmarshal(&bytes).expect("reload checkpoint after crash");
|
||||
let resumed = run_until_finished(
|
||||
&root,
|
||||
durable,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
},
|
||||
12,
|
||||
)
|
||||
.await;
|
||||
let complete = resumed.checked_flatten("bucket").expect("restarted compacted root");
|
||||
assert_eq!((complete.objects, complete.size, complete.versions), (40, 120, 40));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn periodic_nested_snapshot_does_not_publish_shared_compacted_accumulator_under_child_path() {
|
||||
|
||||
@@ -74,7 +74,9 @@ pub(in crate::scanner_folder) async fn ordered_entries(
|
||||
}
|
||||
let mut entries = Vec::new();
|
||||
while let Some(entry) = reader.next_entry().await? {
|
||||
entries.push(entry);
|
||||
// This fixture reorders native entries. Their kernel positions no
|
||||
// longer describe the prefixes delivered to the walker.
|
||||
entries.push(entry.without_anchor());
|
||||
}
|
||||
// Real entries and file types, with an explicit adversarial order rather
|
||||
// than an assumption about the filesystem's enumeration order.
|
||||
@@ -533,6 +535,32 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassifies_on_resume() {
|
||||
check_candidate_resume(CandidateResume::Legacy, 136).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_candidate_warm_resume_avoids_rereading_sealed_root() {
|
||||
check_candidate_resume(CandidateResume::Warm, 135).await;
|
||||
}
|
||||
|
||||
#[cfg(any(unix, windows))]
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn erasure_candidate_cold_resume_uses_the_validated_sealed_root() {
|
||||
check_candidate_resume(CandidateResume::Cold, 135).await;
|
||||
}
|
||||
|
||||
enum CandidateResume {
|
||||
Warm,
|
||||
Legacy,
|
||||
#[cfg(any(unix, windows))]
|
||||
Cold,
|
||||
}
|
||||
|
||||
async fn check_candidate_resume(kind: CandidateResume, expected_io: u64) {
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
|
||||
let (mut scanner, root, folder) = candidate_scanner().await;
|
||||
let _guard = TestGuard {
|
||||
temp_dir: Some(root.clone()),
|
||||
@@ -596,7 +624,33 @@ async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassi
|
||||
.expect("validated candidate index");
|
||||
assert_eq!(index.indexed_entries().expect("index entries").len(), 64);
|
||||
let bytes = serde_json::to_vec(&snapshot).expect("serialize partial checkpoint");
|
||||
let reloaded: DataUsageCache = serde_json::from_slice(&bytes).expect("reload partial checkpoint");
|
||||
let mut reloaded: DataUsageCache = serde_json::from_slice(&bytes).expect("reload partial checkpoint");
|
||||
if !matches!(kind, CandidateResume::Warm) {
|
||||
scanner.directory_checkpoints.clear();
|
||||
if matches!(kind, CandidateResume::Legacy) {
|
||||
// A pre-directory-checkpoint writer persisted raw names but no
|
||||
// authenticated physical position or sealed membership manifest.
|
||||
reloaded.info.scan_directory_source = None;
|
||||
reloaded.info.scan_directory_work.clear();
|
||||
} else {
|
||||
for state in &reloaded.info.scan_directory_work {
|
||||
scanner
|
||||
.directory_checkpoints
|
||||
.entry(state.parent().to_owned())
|
||||
.or_default()
|
||||
.insert(state.endpoint().to_owned(), Arc::new(std::sync::Mutex::new(Some(state.clone()))));
|
||||
}
|
||||
}
|
||||
scanner.local_disk = new_disk(
|
||||
&scanner.local_disk.endpoint(),
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("reopen cold scanner disk");
|
||||
}
|
||||
scanner.old_cache = reloaded.clone();
|
||||
scanner.new_cache = reloaded;
|
||||
scanner.raw_enumeration_progress.clear();
|
||||
@@ -615,8 +669,8 @@ async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassi
|
||||
);
|
||||
assert_eq!(
|
||||
charged_ops(&root),
|
||||
136,
|
||||
"resume recharges actual reads; the raw index is not a filesystem seek"
|
||||
expected_io,
|
||||
"resume charges every physical read, including cold reconstruction of the root"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -774,7 +828,6 @@ async fn erasure_candidate_covered_namespace_reuses_totals_after_classification(
|
||||
},
|
||||
);
|
||||
scanner.budget = budget.clone();
|
||||
let before = global_metrics().scan_observation().snapshot();
|
||||
scanner
|
||||
.scan_folder(
|
||||
budget.token(),
|
||||
@@ -791,15 +844,6 @@ async fn erasure_candidate_covered_namespace_reuses_totals_after_classification(
|
||||
assert_eq!(budget.progress(), (0, 1), "only the parent consumes namespace quota");
|
||||
assert_eq!(charged_ops(&root), 4, "list both directories once without re-reading covered metadata");
|
||||
assert_eq!(scanner.new_cache.find(&folder.name).expect("reused candidate").objects, 1);
|
||||
let observed = global_metrics().scan_observation().snapshot();
|
||||
assert_eq!(
|
||||
observed.cache_reuse["classified_namespace"].count,
|
||||
before.cache_reuse["classified_namespace"].count + 1
|
||||
);
|
||||
assert_eq!(
|
||||
observed.metadata_reads.attempts, before.metadata_reads.attempts,
|
||||
"classification retains the existing zero metadata read path"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -879,7 +923,6 @@ async fn erasure_candidate_preserves_compacted_namespace_sampling_after_classifi
|
||||
},
|
||||
);
|
||||
scanner.budget = budget.clone();
|
||||
let before = global_metrics().scan_observation().snapshot();
|
||||
scanner
|
||||
.scan_folder(
|
||||
budget.token(),
|
||||
@@ -900,15 +943,6 @@ async fn erasure_candidate_preserves_compacted_namespace_sampling_after_classifi
|
||||
"sampling still requires full current classification, without metadata re-read"
|
||||
);
|
||||
assert_eq!(scanner.new_cache.find(&folder.name).expect("sampled candidate").objects, 1);
|
||||
let observed = global_metrics().scan_observation().snapshot();
|
||||
assert_eq!(
|
||||
observed.cache_reuse["classified_namespace"].count,
|
||||
before.cache_reuse["classified_namespace"].count + 1
|
||||
);
|
||||
assert_eq!(
|
||||
observed.metadata_reads.attempts, before.metadata_reads.attempts,
|
||||
"classification retains the existing zero metadata read path"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -4,6 +4,25 @@
|
||||
use super::enumeration_restart::{install_raw_entry_observer, observed_raw_entries, seed_flat_bucket};
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudgetConfig;
|
||||
use std::cell::Cell;
|
||||
use std::path::PathBuf;
|
||||
|
||||
const PHYSICAL_READ_COST: Duration = Duration::from_secs(60);
|
||||
|
||||
tokio::task_local! {
|
||||
static SLOW_DIRECTORY: (PathBuf, Cell<usize>);
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) fn is_slow_directory(path: &str) -> bool {
|
||||
SLOW_DIRECTORY.try_with(|(directory, _)| directory.as_path() == std::path::Path::new(path)) == Ok(true)
|
||||
}
|
||||
|
||||
/// Charge every read attempt (including replay and EOF) before entries leave
|
||||
/// the native buffer, so cancelling a simulated slow read cannot skip a name.
|
||||
pub(in crate::scanner_folder) async fn charge_physical_read() {
|
||||
SLOW_DIRECTORY.with(|(_, reads)| reads.set(reads.get() + 1));
|
||||
tokio::time::advance(PHYSICAL_READ_COST).await;
|
||||
}
|
||||
|
||||
async fn open_fixture_disk(disk_root: &std::path::Path) -> Arc<Disk> {
|
||||
let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("fixture endpoint");
|
||||
@@ -18,6 +37,73 @@ async fn open_fixture_disk(disk_root: &std::path::Path) -> Arc<Disk> {
|
||||
.expect("open synthetic disk")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn wide_directory_runtime_budget_reaches_eof() {
|
||||
const WIDTH: usize = 48;
|
||||
const ROUNDS: usize = 16;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&disk_root, WIDTH).await;
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let mut anchor = FileInfo::new("large", 4, 2);
|
||||
anchor.volume = "bucket".to_owned();
|
||||
anchor.version_id = Some(Uuid::from_u128(1000));
|
||||
anchor.versioned = true;
|
||||
anchor.mod_time = Some(OffsetDateTime::now_utc());
|
||||
anchor.size = 4096;
|
||||
let mut metadata = FileMeta::new();
|
||||
metadata.add_version(anchor).expect("large fixture object");
|
||||
write_test_object_metadata_bytes(&disk_root, "bucket", "large", &metadata.marshal_msg().expect("encode large object")).await;
|
||||
let parent = CancellationToken::new();
|
||||
let expected = (WIDTH + 1, WIDTH + 4096);
|
||||
let scan = async {
|
||||
for prior_timeouts in 0..ROUNDS {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(8 * PHYSICAL_READ_COST),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
cache.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(complete) => {
|
||||
assert!(prior_timeouts >= 2, "the wide directory must span multiple actual runtime deadlines");
|
||||
assert!(complete.info.snapshot_complete);
|
||||
let totals = complete.checked_flatten("bucket").expect("complete bucket root");
|
||||
assert_eq!((totals.objects, totals.size), expected);
|
||||
return;
|
||||
}
|
||||
Err(ScannerError::PartialCache(partial)) => {
|
||||
assert!(!partial.info.snapshot_complete);
|
||||
cache = DataUsageCache::unmarshal(&partial.marshal_msg().expect("persist partial")).expect("reload partial");
|
||||
}
|
||||
Err(ScannerError::Other(message)) if budget.token().is_cancelled() && message == "Operation cancelled" => {}
|
||||
Err(error) => panic!("unexpected scan failure: {error}"),
|
||||
}
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Runtime));
|
||||
}
|
||||
let reads = SLOW_DIRECTORY.with(|(_, count)| count.get());
|
||||
let totals = cache.checked_flatten("bucket").expect("last partial root");
|
||||
panic!(
|
||||
"wide directory still incomplete after {ROUNDS} fixed runtime deadlines and {reads} physical reads: {} objects, {} bytes",
|
||||
totals.objects, totals.size
|
||||
);
|
||||
};
|
||||
SLOW_DIRECTORY.scope((disk.path().join("bucket"), Cell::new(0)), scan).await;
|
||||
}
|
||||
|
||||
/// A count budget is the operator's total work limit. It must also stop the
|
||||
/// first listing of a directory far wider than that budget, instead of letting
|
||||
/// one read_dir pass finish before the first object is charged.
|
||||
@@ -75,15 +161,28 @@ async fn scan_one_restart_round(
|
||||
disk_root: &std::path::Path,
|
||||
cache: DataUsageCache,
|
||||
config: ScannerCycleBudgetConfig,
|
||||
) -> RestartRound {
|
||||
scan_one_replica_round(disk_root, None, cache, config).await
|
||||
}
|
||||
|
||||
async fn scan_one_replica_round(
|
||||
disk_root: &std::path::Path,
|
||||
peer: Option<&std::path::Path>,
|
||||
cache: DataUsageCache,
|
||||
config: ScannerCycleBudgetConfig,
|
||||
) -> RestartRound {
|
||||
let disk = open_fixture_disk(disk_root).await;
|
||||
let mut disks = vec![disk.clone()];
|
||||
if let Some(peer) = peer {
|
||||
disks.push(open_fixture_disk(peer).await);
|
||||
}
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, config);
|
||||
let _observation = install_raw_entry_observer(disk.path());
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![disk.clone()],
|
||||
disks,
|
||||
disk,
|
||||
cache.clone(),
|
||||
None,
|
||||
@@ -105,7 +204,7 @@ async fn scan_one_restart_round(
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_until_finished(
|
||||
pub(super) async fn run_until_finished(
|
||||
disk_root: &std::path::Path,
|
||||
mut cache: DataUsageCache,
|
||||
config: ScannerCycleBudgetConfig,
|
||||
@@ -231,3 +330,476 @@ async fn fair_prefix_quantum_budget_stop_keeps_progress_without_certifying_it()
|
||||
let expected = u64::try_from(OBJECTS).expect("fixture size");
|
||||
assert_eq!((totals.objects as u64, totals.versions as u64), (expected, expected));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_compacted_directory_totals_progress_after_every_executor_restart() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&root, 40).await;
|
||||
let budget = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
let mut partial_counts = Vec::new();
|
||||
for round in 0..16 {
|
||||
let step = scan_one_restart_round(&root, cache.clone(), budget).await;
|
||||
if !step.finished {
|
||||
assert!(
|
||||
step.outcome
|
||||
.durable_scan_progress()
|
||||
.advanced_from(&cache.durable_scan_progress()),
|
||||
"each cold compacted round must earn a coordinator service turn, round {round}"
|
||||
);
|
||||
}
|
||||
cache = step.outcome;
|
||||
let total = cache.checked_flatten("bucket").expect("connected compact bucket");
|
||||
if step.finished {
|
||||
assert!(round > 1);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
assert_eq!((total.objects, total.size, total.versions), (40, 40, 40));
|
||||
return;
|
||||
}
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
partial_counts.push(total.objects);
|
||||
}
|
||||
panic!("cold compacted scan did not finish in 16 rounds; objects per partial: {partial_counts:?}");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_compacted_replicas_finish_within_their_independent_object_budgets() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let first = temp.path().join("a");
|
||||
let other = temp.path().join("b");
|
||||
let mut cache = seed_flat_bucket(&first, 40).await;
|
||||
tokio::fs::create_dir_all(&other).await.expect("fresh replica");
|
||||
for index in (0..40).rev() {
|
||||
let name = format!("object-{index:04}");
|
||||
let metadata = tokio::fs::read(first.join("bucket").join(&name).join("xl.meta"))
|
||||
.await
|
||||
.expect("original metadata");
|
||||
write_test_object_metadata_bytes(&other, "bucket", &name, &metadata).await;
|
||||
}
|
||||
let budget = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
for round in 0..(2 * 40_usize.div_ceil(6) + 1) {
|
||||
let root = if round % 2 == 0 { &first } else { &other };
|
||||
let peer = if round % 2 == 0 { &other } else { &first };
|
||||
let step = scan_one_replica_round(root, Some(peer), cache.clone(), budget).await;
|
||||
if !step.finished {
|
||||
assert!(
|
||||
step.outcome
|
||||
.durable_scan_progress()
|
||||
.advanced_from(&cache.durable_scan_progress()),
|
||||
"cross-disk compacted work should advance durable scheduling, round {round}"
|
||||
);
|
||||
}
|
||||
cache = step.outcome;
|
||||
if step.finished {
|
||||
assert!(round > 1);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
let total = cache.checked_flatten("bucket").expect("complete switched source");
|
||||
assert_eq!((total.objects, total.size), (40, 40));
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("two cold physical replicas did not converge within two independent object work budgets");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_compacted_identical_names_do_not_mix_replica_version_sizes() {
|
||||
use super::checkpoint_fixture::write_checkpoint_object;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let a = temp.path().join("a");
|
||||
let b = temp.path().join("b");
|
||||
let start = seed_flat_bucket(&a, 40).await;
|
||||
seed_flat_bucket(&b, 40).await;
|
||||
write_checkpoint_object(&b, "object-0000", &[(Some(Uuid::from_u128(100)), 11), (Some(Uuid::from_u128(101)), 23)]).await;
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
let first = scan_one_replica_round(&a, Some(&b), start, config).await;
|
||||
assert!(!first.finished);
|
||||
let mut cache = first.outcome;
|
||||
for _ in 0..10 {
|
||||
let round = scan_one_replica_round(&b, Some(&a), cache, config).await;
|
||||
cache = round.outcome;
|
||||
if round.finished {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let complete = cache;
|
||||
assert!(complete.info.snapshot_complete);
|
||||
let counted = complete.checked_flatten("bucket").expect("complete B accounting");
|
||||
// 39 one-byte objects and both retained versions of the remaining key.
|
||||
assert_eq!((counted.objects, counted.size, counted.versions), (40, 73, 41));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn legacy_unbound_partial_restarts_on_the_actual_physical_replica() {
|
||||
use super::checkpoint_fixture::write_checkpoint_object;
|
||||
let temp = tempfile::tempdir().expect("fixture");
|
||||
let a = temp.path().join("a");
|
||||
let b = temp.path().join("b");
|
||||
let cache = seed_flat_bucket(&a, 40).await;
|
||||
seed_flat_bucket(&b, 40).await;
|
||||
write_checkpoint_object(&b, "object-0000", &[(Some(Uuid::from_u128(200)), 12), (Some(Uuid::from_u128(201)), 31)]).await;
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
let first = scan_one_restart_round(&a, cache, config).await;
|
||||
let mut legacy = first.outcome;
|
||||
assert!(legacy.validated_scan_frontier().is_some());
|
||||
legacy.info.scan_directory_source = None;
|
||||
legacy.info.scan_directory_work.clear();
|
||||
legacy.info.scan_source_trees.clear();
|
||||
let upgraded = run_until_finished(&b, legacy, config, 10).await;
|
||||
let total = upgraded.checked_flatten("bucket").expect("B complete");
|
||||
assert_eq!((total.objects, total.size, total.versions), (40, 82, 41));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_linked_prefixes_resume_their_own_replica_coverage() {
|
||||
use super::checkpoint_fixture::write_checkpoint_object;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let a = temp.path().join("a");
|
||||
let b = temp.path().join("b");
|
||||
let mut cache = seed_flat_bucket(&a, 40).await;
|
||||
seed_flat_bucket(&b, 40).await;
|
||||
write_checkpoint_object(&b, "object-0000", &[(Some(Uuid::from_u128(100)), 11), (Some(Uuid::from_u128(101)), 23)]).await;
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
// Six turns per disk leave one final B quantum. Each cold process must
|
||||
// recover that disk's linked tree as well as its complete child coverage.
|
||||
for round in 0..(2 * 40_usize.div_ceil(6) + 1) {
|
||||
let (root, peer) = if round < 12 && round % 2 == 0 { (&a, &b) } else { (&b, &a) };
|
||||
let previous_source = cache.info.scan_directory_source.clone();
|
||||
let before = cache.durable_scan_progress();
|
||||
let result = scan_one_replica_round(root, Some(peer), cache, config).await;
|
||||
if !result.finished {
|
||||
assert!(
|
||||
result.outcome.durable_scan_progress().advanced_from(&before),
|
||||
"linked source must earn another turn, round {round}"
|
||||
);
|
||||
}
|
||||
cache = result.outcome;
|
||||
if round == 1 {
|
||||
let previous_source = previous_source.expect("A's physical source");
|
||||
let endpoints = [
|
||||
previous_source.as_str(),
|
||||
cache.info.scan_directory_source.as_deref().expect("B's physical source"),
|
||||
]
|
||||
.map(|source| source.split_once('|').expect("disk generation and endpoint").1.to_owned())
|
||||
.into_iter()
|
||||
.collect();
|
||||
let mut only_switched = cache.clone();
|
||||
only_switched.activate_directory_source(previous_source, &endpoints);
|
||||
assert!(only_switched.validated_scan_frontier().is_some(), "A's linked coverage is retained");
|
||||
assert!(
|
||||
!only_switched
|
||||
.durable_scan_progress()
|
||||
.advanced_from(&cache.durable_scan_progress()),
|
||||
"moving a linked source between active and dormant earns no new turn"
|
||||
);
|
||||
}
|
||||
|
||||
if result.finished {
|
||||
assert_eq!(root, &b);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
let root = cache.find("bucket").expect("complete linked bucket");
|
||||
assert!(!root.compacted);
|
||||
assert_eq!(root.children.len(), 40);
|
||||
let counted = cache.checked_flatten("bucket").expect("full B view");
|
||||
assert_eq!((counted.objects, counted.size, counted.versions), (40, 73, 41));
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("two independent cold linked prefix scans did not converge");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn cold_compacted_changed_replica_does_not_import_another_disks_totals() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let first = temp.path().join("a");
|
||||
let other = temp.path().join("b");
|
||||
let baseline = seed_flat_bucket(&first, 40).await;
|
||||
seed_flat_bucket(&other, 40).await;
|
||||
// Object zero was already counted on A but never existed in B's view.
|
||||
// Importing A's subtotal based only on a matching child boundary would
|
||||
// produce 40 objects on B even though its authoritative listing has 39.
|
||||
tokio::fs::remove_dir_all(other.join("bucket/object-0000"))
|
||||
.await
|
||||
.expect("different replica membership");
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
};
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
let first = scan_one_restart_round(&first, baseline, config).await;
|
||||
assert!(!first.finished);
|
||||
let complete = run_until_finished(&other, first.outcome, config, 14).await;
|
||||
let root = complete.checked_flatten("bucket").expect("finished replica");
|
||||
assert_eq!((root.objects, root.versions, root.size), (39, 39, 39));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn cold_wide_listing_rotates_physical_endpoints_before_eof() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let first = temp.path().join("a");
|
||||
let other = temp.path().join("b");
|
||||
let mut cache = seed_flat_bucket(&first, 49).await;
|
||||
seed_flat_bucket(&other, 49).await;
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_duration: Some(8 * PHYSICAL_READ_COST),
|
||||
..Default::default()
|
||||
};
|
||||
for round in 0..22 {
|
||||
let disk = if round % 2 == 0 { &first } else { &other };
|
||||
let peer = if round % 2 == 0 { &other } else { &first };
|
||||
let physical = disk.join("bucket").canonicalize().expect("physical directory");
|
||||
let before = cache.durable_scan_progress();
|
||||
let step = SLOW_DIRECTORY
|
||||
.scope((physical, Cell::new(0)), scan_one_replica_round(disk, Some(peer), cache, config))
|
||||
.await;
|
||||
if !step.finished {
|
||||
assert!(
|
||||
step.outcome.durable_scan_progress().advanced_from(&before),
|
||||
"each independent native resume must advance durable scheduling, round {round}"
|
||||
);
|
||||
}
|
||||
cache = step.outcome;
|
||||
if step.finished {
|
||||
assert!(round > 2);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
let total = cache.checked_flatten("bucket").expect("switched cold listing");
|
||||
assert_eq!((total.objects, total.size), (49, 49));
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("rotating before EOF repeatedly overwrote an endpoint's physical progress");
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "macos"))]
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial]
|
||||
async fn cold_wide_listing_reaches_eof_after_every_executor_restart() {
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&root, 49).await;
|
||||
let budget = ScannerCycleBudgetConfig {
|
||||
max_duration: Some(8 * PHYSICAL_READ_COST),
|
||||
..Default::default()
|
||||
};
|
||||
SLOW_DIRECTORY
|
||||
.scope((root.join("bucket").canonicalize().expect("physical path"), Cell::new(0)), async {
|
||||
for round in 0..16 {
|
||||
let step = scan_one_restart_round(&root, cache.clone(), budget).await;
|
||||
cache = step.outcome;
|
||||
if step.finished {
|
||||
assert!(round > 1);
|
||||
assert!(cache.info.snapshot_complete);
|
||||
let total = cache.checked_flatten("bucket").expect("complete bucket");
|
||||
assert_eq!((total.objects, total.size), (49, 49));
|
||||
return;
|
||||
}
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
}
|
||||
let reads = SLOW_DIRECTORY.with(|(_, reads)| reads.get());
|
||||
panic!("cold physical listing did not reach EOF after 16 rounds and {reads} directory reads");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_compacted_directory_reuses_only_finished_children() {
|
||||
const WIDTH: usize = 40;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let mut cache = seed_flat_bucket(&disk_root, WIDTH).await;
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let parent = CancellationToken::new();
|
||||
super::super::TEST_FORCE_COMPACTION_WIDTH
|
||||
.scope(16, async {
|
||||
for round in 0..16 {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects: Some(6),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
cache.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(completed) => {
|
||||
assert!(round > 1);
|
||||
assert!(completed.info.snapshot_complete);
|
||||
let root = completed.find("bucket").expect("compacted bucket");
|
||||
assert!(root.compacted);
|
||||
assert_eq!((root.objects, root.size), (WIDTH, WIDTH));
|
||||
return;
|
||||
}
|
||||
Err(ScannerError::PartialCache(partial)) => {
|
||||
cache = DataUsageCache::unmarshal(&partial.marshal_msg().expect("save compacted checkpoint"))
|
||||
.expect("reload partial");
|
||||
}
|
||||
Err(error) => panic!("unexpected compacted scan failure: {error}"),
|
||||
}
|
||||
}
|
||||
panic!("fixed object budget failed to finish compacted children");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_directory_replacement_invalidates_parked_entries() {
|
||||
const WIDTH: usize = 48;
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let initial = seed_flat_bucket(&disk_root, WIDTH).await;
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let parent = CancellationToken::new();
|
||||
let config = ScannerCycleBudgetConfig {
|
||||
max_entries: Some(3),
|
||||
..Default::default()
|
||||
};
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, config);
|
||||
let Err(ScannerError::PartialCache(partial)) = scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
initial,
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await
|
||||
else {
|
||||
panic!("the original listing must stop before EOF")
|
||||
};
|
||||
assert!(
|
||||
super::super::parked_bucket_disks("bucket")
|
||||
.iter()
|
||||
.any(|scope| scope.ptr_eq(&Arc::downgrade(&disk)))
|
||||
);
|
||||
let mut cache = *partial;
|
||||
// Rename pins the old inode. Reusing its open stream or its name pages
|
||||
// would visit the vanished objects and omit this replacement's metadata.
|
||||
tokio::fs::rename(disk_root.join("bucket"), temp.path().join("previous"))
|
||||
.await
|
||||
.expect("move old namespace");
|
||||
let metadata = metadata_for_object_version("bucket", "successor", Some(Uuid::from_u128(100)));
|
||||
write_test_object_metadata_bytes(&disk_root, "bucket", "successor", &metadata).await;
|
||||
for _ in 0..6 {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, config);
|
||||
match scan_data_folder(
|
||||
budget.token(),
|
||||
budget,
|
||||
vec![disk.clone()],
|
||||
disk.clone(),
|
||||
cache.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(complete) => {
|
||||
assert!(complete.info.snapshot_complete);
|
||||
let total = complete.checked_flatten("bucket").expect("current root");
|
||||
assert_eq!((total.objects, total.size), (1, 1));
|
||||
assert!(complete.find("bucket/successor").is_some());
|
||||
assert!(
|
||||
!super::super::parked_bucket_disks("bucket")
|
||||
.iter()
|
||||
.any(|scope| scope.ptr_eq(&Arc::downgrade(&disk)))
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(ScannerError::PartialCache(partial)) => cache = *partial,
|
||||
Err(error) => panic!("replacement scan failed: {error}"),
|
||||
}
|
||||
}
|
||||
panic!("replacement directory never finished");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn wide_directory_deleted_bucket_releases_a_slot_under_pressure() {
|
||||
use super::super::directory_cursor::{DirectoryScope, PARKED_DIRECTORIES_PER_DISK, ScannerDirectory, is_parked};
|
||||
|
||||
let temp = tempfile::tempdir().expect("fixture directory");
|
||||
let disk_root = temp.path().join("disk");
|
||||
let cache = seed_flat_bucket(&disk_root, 0).await;
|
||||
let disk = open_fixture_disk(&disk_root).await;
|
||||
let scope = DirectoryScope::new(&disk, &cache, FolderScanMode::Namespace, None).expect("physical source and owner");
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default());
|
||||
budget.cancel_for_runtime();
|
||||
let retired = disk_root.join("retired");
|
||||
for index in 0..PARKED_DIRECTORIES_PER_DISK {
|
||||
let path = retired.join(index.to_string());
|
||||
tokio::fs::create_dir_all(&path).await.expect("create parked directory");
|
||||
let path = path.to_str().expect("fixture path");
|
||||
let directory = ScannerDirectory::open(path, "bucket", Some(scope.clone()), budget.clone(), None)
|
||||
.await
|
||||
.expect("open source");
|
||||
drop(directory);
|
||||
assert!(is_parked(path));
|
||||
}
|
||||
let next = disk_root.join("replacement");
|
||||
tokio::fs::create_dir(&next).await.expect("current source");
|
||||
let next = next.to_str().expect("fixture path");
|
||||
let directory = ScannerDirectory::open(next, "bucket", Some(scope.clone()), budget.clone(), None)
|
||||
.await
|
||||
.expect("open next source");
|
||||
drop(directory);
|
||||
assert!(!is_parked(next), "active owners retain their bounded slots");
|
||||
tokio::fs::remove_dir_all(retired).await.expect("retire directories");
|
||||
let directory = ScannerDirectory::open(next, "bucket", Some(scope), budget, None)
|
||||
.await
|
||||
.expect("admit after retirement");
|
||||
drop(directory);
|
||||
assert!(is_parked(next), "an obsolete source must not indefinitely exclude new work");
|
||||
super::super::directory_cursor::forget_bucket(&disk, &cache.info);
|
||||
}
|
||||
|
||||
@@ -40,7 +40,6 @@ use rustfs_lock::{LockError, NamespaceLockGuard};
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
|
||||
};
|
||||
use rustfs_scanner_metrics::scan_observation::{PREFIX_REUSE_BLOCKERS, ScannerBaselineRejection, ScannerPrefixReuseStage};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
@@ -113,7 +112,16 @@ pub(crate) const SCANNER_SEGMENT_ACTIVATION_PROOF_INPUTS: [&str; 9] = [
|
||||
"durable_dirty_producer_journal",
|
||||
"ack_generation_guard",
|
||||
];
|
||||
pub(crate) const SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS: [&str; 8] = PREFIX_REUSE_BLOCKERS;
|
||||
pub(crate) const SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS: [&str; 8] = [
|
||||
"missing_producer_identity",
|
||||
"missing_durable_journal_replay",
|
||||
"restart_gap",
|
||||
"generation_gap",
|
||||
"overflow",
|
||||
"stale_ack_generation",
|
||||
"missing_cold_zero_walk_oracle",
|
||||
"distributed_without_peer_invalidation",
|
||||
];
|
||||
|
||||
pub type DirtyUsageBuckets = HashMap<String, u64>;
|
||||
|
||||
@@ -189,7 +197,6 @@ struct VerifiedRemoteDirtyUsage {
|
||||
}
|
||||
|
||||
struct ScannerBucketScopeResolutionResult {
|
||||
segment_reuse_activation_preflight: Option<ScannerSegmentReuseActivationPreflight>,
|
||||
scope: ScannerBucketScanScope,
|
||||
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
|
||||
distributed_segment_invalidation_evidence: Option<DistributedSegmentInvalidationEvidence>,
|
||||
@@ -229,13 +236,6 @@ pub(crate) struct ScannerSegmentReuseActivationPreflight {
|
||||
}
|
||||
|
||||
impl ScannerSegmentReuseActivationPreflight {
|
||||
fn record_observation(&self, stage: ScannerPrefixReuseStage) {
|
||||
global_metrics().scan_observation().record_prefix_reuse(
|
||||
stage,
|
||||
self.scanner_segment_reuse_activated,
|
||||
&self.fail_closed_blockers,
|
||||
);
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub(crate) fn fail_closed_blockers(&self) -> impl Iterator<Item = &'static str> + '_ {
|
||||
self.fail_closed_blockers.iter().filter_map(|blocker| *blocker)
|
||||
@@ -319,7 +319,6 @@ fn resolve_remote_dirty_usage_scope(
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
) -> ScannerBucketScopeResolutionResult {
|
||||
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
|
||||
segment_reuse_activation_preflight: None,
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: Vec::new(),
|
||||
distributed_segment_invalidation_evidence: None,
|
||||
@@ -411,7 +410,6 @@ fn resolve_remote_dirty_usage_scope(
|
||||
};
|
||||
|
||||
ScannerBucketScopeResolutionResult {
|
||||
segment_reuse_activation_preflight: Some(segment_reuse_activation_preflight),
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: scoped_acknowledgements,
|
||||
distributed_segment_invalidation_evidence,
|
||||
@@ -429,81 +427,43 @@ fn complete_scanner_cache_snapshot_plan_digest(
|
||||
proof: ScannerCacheBaselineProof<'_>,
|
||||
expected_converged: Option<bool>,
|
||||
) -> Option<DataUsageScanPlanDigest> {
|
||||
validate_scanner_cache_snapshot_plan_digest(snapshot, proof, expected_converged).ok()
|
||||
}
|
||||
|
||||
fn validate_scanner_cache_snapshot_plan_digest(
|
||||
snapshot: &DataUsageInfo,
|
||||
proof: ScannerCacheBaselineProof<'_>,
|
||||
expected_converged: Option<bool>,
|
||||
) -> std::result::Result<DataUsageScanPlanDigest, ScannerBaselineRejection> {
|
||||
// The observational mark disqualifies only the authoritative slot: an
|
||||
// observed candidate carries it by construction.
|
||||
let marks_accepted = match expected_converged {
|
||||
Some(expected) => snapshot.usage_snapshot_converged == Some(expected),
|
||||
None => snapshot.usage_snapshot_converged.is_some() && !snapshot.usage_snapshot_observational,
|
||||
};
|
||||
if !snapshot.is_complete_bucket_usage_snapshot() {
|
||||
return Err(ScannerBaselineRejection::Incomplete);
|
||||
}
|
||||
if snapshot.usage_snapshot_partial {
|
||||
return Err(ScannerBaselineRejection::Partial);
|
||||
}
|
||||
if !marks_accepted {
|
||||
return Err(ScannerBaselineRejection::Marks);
|
||||
}
|
||||
if snapshot.scanner_epoch != Some(proof.leader_epoch) {
|
||||
return Err(ScannerBaselineRejection::Epoch);
|
||||
}
|
||||
if snapshot.usage_snapshot_set_states.len() != proof.expected_sources.len() {
|
||||
return Err(ScannerBaselineRejection::SourceCount);
|
||||
if !snapshot.is_complete_bucket_usage_snapshot()
|
||||
|| snapshot.usage_snapshot_partial
|
||||
|| !marks_accepted
|
||||
|| snapshot.scanner_epoch != Some(proof.leader_epoch)
|
||||
|| snapshot.usage_snapshot_set_states.len() != proof.expected_sources.len()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
// Completed maintenance also covers ordinary usage. Keep its exact stored
|
||||
// proof for cache reuse, and reject mixtures of different set work proofs.
|
||||
let baseline_plan_digest = DataUsageScanPlanDigest(
|
||||
snapshot
|
||||
.usage_snapshot_set_states
|
||||
.first()
|
||||
.and_then(|state| state.scan_plan_digest)
|
||||
.ok_or(ScannerBaselineRejection::PlanMissing)?,
|
||||
);
|
||||
let baseline_plan_digest = DataUsageScanPlanDigest(snapshot.usage_snapshot_set_states.first()?.scan_plan_digest?);
|
||||
if !scanner_plan_digest_is_work_variant(proof.scan_plan_digest, baseline_plan_digest) {
|
||||
return Err(ScannerBaselineRejection::PlanMismatch);
|
||||
return None;
|
||||
}
|
||||
let mut states = HashSet::with_capacity(snapshot.usage_snapshot_set_states.len());
|
||||
for state in &snapshot.usage_snapshot_set_states {
|
||||
let source = DataUsageCacheSource::new(
|
||||
usize::try_from(state.pool_index).map_err(|_| ScannerBaselineRejection::SourceIndex)?,
|
||||
usize::try_from(state.set_index).map_err(|_| ScannerBaselineRejection::SourceIndex)?,
|
||||
);
|
||||
if !proof.expected_sources.contains(&source) {
|
||||
return Err(ScannerBaselineRejection::SourceMembership);
|
||||
}
|
||||
if !states.insert(source) {
|
||||
return Err(ScannerBaselineRejection::SourceDuplicate);
|
||||
}
|
||||
if !state.complete {
|
||||
return Err(ScannerBaselineRejection::SetIncomplete);
|
||||
}
|
||||
if state.tombstone {
|
||||
return Err(ScannerBaselineRejection::SetTombstone);
|
||||
}
|
||||
if state.scanner_epoch != Some(proof.leader_epoch) {
|
||||
return Err(ScannerBaselineRejection::SetEpoch);
|
||||
}
|
||||
if state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle) {
|
||||
return Err(ScannerBaselineRejection::SetCycle);
|
||||
}
|
||||
if state.scan_plan_digest != Some(baseline_plan_digest.0) {
|
||||
return Err(ScannerBaselineRejection::SetPlan);
|
||||
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
|
||||
if !proof.expected_sources.contains(&source)
|
||||
|| !states.insert(source)
|
||||
|| !state.complete
|
||||
|| state.tombstone
|
||||
|| state.scanner_epoch != Some(proof.leader_epoch)
|
||||
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|
||||
|| state.scan_plan_digest != Some(baseline_plan_digest.0)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
if states != *proof.expected_sources {
|
||||
return Err(ScannerBaselineRejection::SourceCoverage);
|
||||
}
|
||||
Ok(baseline_plan_digest)
|
||||
(states == *proof.expected_sources).then_some(baseline_plan_digest)
|
||||
}
|
||||
|
||||
/// Whether `digest` is `structural` under any work flag: the same bucket
|
||||
@@ -533,35 +493,10 @@ fn scanner_baseline_pending_buckets(proof: &ScannerCacheBaselineProof<'_>) -> Ve
|
||||
}
|
||||
|
||||
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
|
||||
validate_scanner_cache_baseline_plan_digest(proof).ok()
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct ScannerBaselineValidationFailure {
|
||||
authoritative: ScannerBaselineRejection,
|
||||
observed: Option<ScannerBaselineRejection>,
|
||||
}
|
||||
|
||||
fn validate_scanner_cache_baseline_plan_digest(
|
||||
proof: ScannerCacheBaselineProof<'_>,
|
||||
) -> std::result::Result<DataUsageScanPlanDigest, ScannerBaselineValidationFailure> {
|
||||
let authoritative_error = |authoritative| ScannerBaselineValidationFailure {
|
||||
authoritative,
|
||||
observed: None,
|
||||
};
|
||||
let authoritative_data = proof
|
||||
.authoritative_data
|
||||
.ok_or_else(|| authoritative_error(ScannerBaselineRejection::Missing))?;
|
||||
let authoritative = serde_json::from_slice::<DataUsageInfo>(authoritative_data)
|
||||
.map_err(|_| authoritative_error(ScannerBaselineRejection::Decode))?;
|
||||
let authoritative_rejection = match validate_scanner_cache_snapshot_plan_digest(&authoritative, proof, None) {
|
||||
Ok(validated_digest) => return Ok(validated_digest),
|
||||
Err(reason) => reason,
|
||||
};
|
||||
let observed_error = |observed| ScannerBaselineValidationFailure {
|
||||
authoritative: authoritative_rejection,
|
||||
observed: Some(observed),
|
||||
};
|
||||
let authoritative = serde_json::from_slice::<DataUsageInfo>(proof.authoritative_data?).ok()?;
|
||||
if let Some(validated_digest) = complete_scanner_cache_snapshot_plan_digest(&authoritative, proof, None) {
|
||||
return Some(validated_digest);
|
||||
}
|
||||
|
||||
// A complete but superseded observation may reuse its per-set cache only
|
||||
// when it was explicitly tied to the durable authoritative baseline. It
|
||||
@@ -571,18 +506,14 @@ fn validate_scanner_cache_baseline_plan_digest(
|
||||
&& !authoritative.usage_snapshot_observational)
|
||||
|| crate::scanner::data_usage_info_is_bootstrap_pending(&authoritative);
|
||||
if !authoritative_has_identity {
|
||||
return Err(observed_error(ScannerBaselineRejection::AuthoritativeIdentity));
|
||||
return None;
|
||||
}
|
||||
let observed_data = proof
|
||||
.observed_candidate_data
|
||||
.ok_or_else(|| observed_error(ScannerBaselineRejection::Missing))?;
|
||||
let observed =
|
||||
serde_json::from_slice::<DataUsageInfo>(observed_data).map_err(|_| observed_error(ScannerBaselineRejection::Decode))?;
|
||||
let observed = serde_json::from_slice::<DataUsageInfo>(proof.observed_candidate_data?).ok()?;
|
||||
if !observed_data_usage_is_newer(&observed, &authoritative) {
|
||||
return Err(observed_error(ScannerBaselineRejection::NotNewer));
|
||||
return None;
|
||||
}
|
||||
|
||||
validate_scanner_cache_snapshot_plan_digest(&observed, proof, Some(false)).map_err(observed_error)
|
||||
complete_scanner_cache_snapshot_plan_digest(&observed, proof, Some(false))
|
||||
}
|
||||
|
||||
fn scanner_segment_reuse_baseline_producer_evidence(
|
||||
@@ -727,12 +658,6 @@ impl ScannerScopeFallbackReason {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ScannerScopeFallback {
|
||||
reason: ScannerScopeFallbackReason,
|
||||
baseline_rejection: Option<ScannerBaselineValidationFailure>,
|
||||
}
|
||||
|
||||
/// Classify a resolved full-namespace scope against the preconditions
|
||||
/// `resolve_scanner_bucket_scan_scope` checks, in the same order. Returns
|
||||
/// `None` when the cycle was required to walk everything or was scoped.
|
||||
@@ -743,24 +668,18 @@ fn scanner_scope_fallback_reason(
|
||||
all_buckets: &[BucketInfo],
|
||||
baseline_proof: ScannerCacheBaselineProof<'_>,
|
||||
distributed: bool,
|
||||
) -> Option<ScannerScopeFallback> {
|
||||
let fallback = |reason| {
|
||||
Some(ScannerScopeFallback {
|
||||
reason,
|
||||
baseline_rejection: None,
|
||||
})
|
||||
};
|
||||
) -> Option<ScannerScopeFallbackReason> {
|
||||
if !scope_eligible || !resolved_scope.is_default() {
|
||||
return None;
|
||||
}
|
||||
if dirty_usage_snapshot.generation == u64::MAX {
|
||||
return fallback(ScannerScopeFallbackReason::DirtyGenerationExhausted);
|
||||
return Some(ScannerScopeFallbackReason::DirtyGenerationExhausted);
|
||||
}
|
||||
if !dirty_usage_snapshot.covers_listed_pending {
|
||||
return fallback(ScannerScopeFallbackReason::DirtySnapshotIncomplete);
|
||||
return Some(ScannerScopeFallbackReason::DirtySnapshotIncomplete);
|
||||
}
|
||||
if dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES {
|
||||
return fallback(ScannerScopeFallbackReason::DirtyBucketOverflow);
|
||||
return Some(ScannerScopeFallbackReason::DirtyBucketOverflow);
|
||||
}
|
||||
// Peers may contribute dirty buckets the local snapshot does not hold.
|
||||
if !distributed {
|
||||
@@ -770,19 +689,13 @@ fn scanner_scope_fallback_reason(
|
||||
.keys()
|
||||
.any(|bucket| listed.contains(bucket.as_str()))
|
||||
{
|
||||
return fallback(ScannerScopeFallbackReason::NoListedDirtyBuckets);
|
||||
return Some(ScannerScopeFallbackReason::NoListedDirtyBuckets);
|
||||
}
|
||||
}
|
||||
if let Err(failure) = validate_scanner_cache_baseline_plan_digest(baseline_proof) {
|
||||
return Some(ScannerScopeFallback {
|
||||
reason: ScannerScopeFallbackReason::BaselineUnavailable,
|
||||
baseline_rejection: Some(failure),
|
||||
});
|
||||
if complete_scanner_cache_baseline_plan_digest(baseline_proof).is_none() {
|
||||
return Some(ScannerScopeFallbackReason::BaselineUnavailable);
|
||||
}
|
||||
distributed.then_some(ScannerScopeFallback {
|
||||
reason: ScannerScopeFallbackReason::PeerScopeUnavailable,
|
||||
baseline_rejection: None,
|
||||
})
|
||||
distributed.then_some(ScannerScopeFallbackReason::PeerScopeUnavailable)
|
||||
}
|
||||
|
||||
fn scanner_segment_reuse_activation_preflight() -> ScannerSegmentReuseActivationPreflight {
|
||||
@@ -1930,6 +1843,7 @@ pub(crate) struct ScannerCycleResult {
|
||||
usage_watch_proofs: Vec<ScannerUsageWatchProof>,
|
||||
usage_watch_retirements: Vec<ScannerUsageWatchProof>,
|
||||
pub(crate) status: ScannerCycleStatus,
|
||||
pub(crate) partial_stage: Option<&'static str>,
|
||||
publication_epoch: Option<u64>,
|
||||
activity_digest: Option<[u8; 32]>,
|
||||
activity_baseline: Option<Arc<crate::scanner::ScannerActivitySnapshot>>,
|
||||
@@ -1956,6 +1870,7 @@ impl ScannerCycleResult {
|
||||
usage_watch_proofs: Vec::new(),
|
||||
usage_watch_retirements: Vec::new(),
|
||||
status,
|
||||
partial_stage: None,
|
||||
publication_epoch: None,
|
||||
activity_digest: None,
|
||||
activity_baseline: None,
|
||||
@@ -1977,6 +1892,11 @@ impl ScannerCycleResult {
|
||||
}
|
||||
}
|
||||
|
||||
fn with_partial_stage(mut self, stage: Option<&'static str>) -> Self {
|
||||
self.partial_stage = stage;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_publication_epoch(mut self, publication_epoch: Option<u64>) -> Self {
|
||||
self.publication_expectation = None;
|
||||
self.publication_epoch = publication_epoch;
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
use super::*;
|
||||
/// scanner cache locks and the cache snapshot persist/publish path.
|
||||
use crate::data_usage_define::{DataUsageCacheBaselineRevisions, clone_cache_value};
|
||||
use crate::data_usage_define::clone_cache_value;
|
||||
use crate::{ScannerConfigObjectDelete, ScannerObjectIO};
|
||||
use rustfs_scanner_metrics::usage_candidate::{
|
||||
ScannerUsageCandidateKind as CandidateKind, ScannerUsageCandidateOutcome as CandidateOutcome,
|
||||
@@ -792,6 +792,7 @@ pub(super) fn completed_data_usage_info(
|
||||
cancelled: bool,
|
||||
) -> Option<(DataUsageInfo, SystemTime)> {
|
||||
completed_usage_candidate(results, scope, tier_registry_names, bucket_plan_complete, cancelled)
|
||||
.ok()
|
||||
.map(|candidate| (candidate.data, candidate.last_update))
|
||||
}
|
||||
|
||||
@@ -810,7 +811,7 @@ pub(super) fn completed_usage_candidate(
|
||||
tier_registry_names: &[String],
|
||||
bucket_plan_complete: bool,
|
||||
cancelled: bool,
|
||||
) -> Option<ValidatedUsageCandidate> {
|
||||
) -> std::result::Result<ValidatedUsageCandidate, CandidateOutcome> {
|
||||
let candidate: std::result::Result<ValidatedUsageCandidate, CandidateOutcome> = (|| {
|
||||
if !bucket_plan_complete {
|
||||
return Err(CandidateOutcome::InventoryIncomplete);
|
||||
@@ -934,7 +935,7 @@ pub(super) fn completed_usage_candidate(
|
||||
Err(outcome) => *outcome,
|
||||
},
|
||||
);
|
||||
candidate.ok()
|
||||
candidate
|
||||
}
|
||||
|
||||
fn tier_accounting_proof_is_publishable(
|
||||
@@ -1494,7 +1495,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
store: Arc<SetDisks>,
|
||||
updates: &mpsc::Sender<DataUsageCache>,
|
||||
mut cache_snapshot: DataUsageCache,
|
||||
initial_revisions: Option<&DataUsageCacheBaselineRevisions>,
|
||||
initial_revisions: Option<&DataUsageCacheRevisions>,
|
||||
cache_cycle_floor: &AtomicU64,
|
||||
expected_publication_epoch: u64,
|
||||
) -> Option<SystemTime> {
|
||||
@@ -1536,10 +1537,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
};
|
||||
|
||||
let mut persisted = DataUsageCache::default();
|
||||
let current_baseline = match persisted
|
||||
.load_with_baseline_revisions(store.clone(), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
{
|
||||
let revisions = match persisted.load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME).await {
|
||||
Ok(revisions) => revisions,
|
||||
Err(err) => {
|
||||
error!(
|
||||
@@ -1561,7 +1559,6 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let revisions = current_baseline.revisions();
|
||||
let Some(scan_plan_digest) = cache_snapshot.info.scan_plan_digest else {
|
||||
log_missing_set_snapshot_metadata(&store, &cache_snapshot.info, "plan_digest");
|
||||
return None;
|
||||
@@ -1625,9 +1622,9 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
{
|
||||
cache_snapshot = persisted;
|
||||
} else {
|
||||
// Another execution must not advance the baseline. A backup catching
|
||||
// up to the same decoded primary changes no baseline content.
|
||||
if !initial_revisions.is_some_and(|initial| current_baseline.permits_unchanged_baseline_since(initial)) {
|
||||
// A later execution may have completed while this scan was walking.
|
||||
// Only replace the cache revision from which this scan started.
|
||||
if initial_revisions != Some(&revisions) {
|
||||
warn!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
|
||||
@@ -1666,7 +1663,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache_snapshot
|
||||
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, revisions, expected_publication_epoch)
|
||||
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
|
||||
@@ -14,7 +14,9 @@
|
||||
use super::*;
|
||||
/// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish.
|
||||
use crate::data_usage_define::clone_cache_value;
|
||||
use rustfs_scanner_metrics::scan_observation::{ScannerBucketReuseKind, ScannerScanObservation};
|
||||
|
||||
mod walker_slots;
|
||||
use walker_slots::AvailableWalkers;
|
||||
|
||||
pub(super) async fn collect_completed_bucket_results(
|
||||
ctx: CancellationToken,
|
||||
@@ -143,32 +145,6 @@ pub(super) struct PreparedScopedSetScan {
|
||||
pub(super) buckets: Vec<BucketInfo>,
|
||||
pub(super) cache: DataUsageCache,
|
||||
pub(super) cold_bucket_reuse_proof: Option<ScopedColdBucketReuseProof>,
|
||||
cold_bucket_reuse_count: u64,
|
||||
cold_bucket_reuse_elapsed: Duration,
|
||||
}
|
||||
|
||||
impl PreparedScopedSetScan {
|
||||
pub(super) fn record_cold_bucket_reuse(&self, observation: &ScannerScanObservation) {
|
||||
observation.record_bucket_reuse(
|
||||
ScannerBucketReuseKind::ScopedColdBucket,
|
||||
self.cold_bucket_reuse_count,
|
||||
self.cold_bucket_reuse_elapsed,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Counts only roots delivered to the collector; cancellation or channel failure is not reuse.
|
||||
pub(super) async fn send_current_cache_root_entry(
|
||||
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
|
||||
root: DataUsageEntryInfo,
|
||||
cache: &DataUsageCache,
|
||||
pending_maintenance_work: &AtomicBool,
|
||||
observation: &ScannerScanObservation,
|
||||
) -> std::result::Result<(), ScannerError> {
|
||||
let started = Instant::now();
|
||||
send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await?;
|
||||
observation.record_bucket_reuse(ScannerBucketReuseKind::CurrentRoot, 1, started.elapsed());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) struct ScopedColdBucketReuseProof {
|
||||
@@ -252,8 +228,6 @@ pub(super) fn prepare_scoped_set_scan(
|
||||
};
|
||||
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
|
||||
let mut current_bucket_names = HashSet::with_capacity(all_buckets.len());
|
||||
let mut cold_bucket_reuse_count = 0u64;
|
||||
let mut cold_bucket_reuse_elapsed = Duration::ZERO;
|
||||
for bucket in all_buckets {
|
||||
if !current_bucket_names.insert(bucket.name.as_str()) {
|
||||
return None;
|
||||
@@ -262,14 +236,11 @@ pub(super) fn prepare_scoped_set_scan(
|
||||
cache.info.scan_bucket_value_origins.remove(&bucket.name);
|
||||
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
} else {
|
||||
let started = Instant::now();
|
||||
cache.copy_with_children(
|
||||
old_cache,
|
||||
&rustfs_data_usage::hash_path(&bucket.name),
|
||||
&Some(rustfs_data_usage::hash_path(DATA_USAGE_ROOT)),
|
||||
);
|
||||
cold_bucket_reuse_count = cold_bucket_reuse_count.saturating_add(1);
|
||||
cold_bucket_reuse_elapsed = cold_bucket_reuse_elapsed.saturating_add(started.elapsed());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -281,8 +252,6 @@ pub(super) fn prepare_scoped_set_scan(
|
||||
.collect(),
|
||||
cache,
|
||||
cold_bucket_reuse_proof,
|
||||
cold_bucket_reuse_count,
|
||||
cold_bucket_reuse_elapsed,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -459,10 +428,7 @@ impl ScannerIOCache for SetDisks {
|
||||
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
|
||||
};
|
||||
let mut old_cache = DataUsageCache::default();
|
||||
let initial_revisions = match old_cache
|
||||
.load_with_baseline_revisions(self.clone(), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
{
|
||||
let initial_revisions = match old_cache.load_with_revisions(self.clone(), DATA_USAGE_CACHE_NAME).await {
|
||||
Ok(revisions) => Some(revisions),
|
||||
Err(e) => {
|
||||
warn!(
|
||||
@@ -592,8 +558,6 @@ impl ScannerIOCache for SetDisks {
|
||||
})
|
||||
});
|
||||
let mut scoped_cache = scoped_scan.map(|mut prepared| {
|
||||
// The copies are completed work even if this turn later fails publication.
|
||||
prepared.record_cold_bucket_reuse(global_metrics().scan_observation());
|
||||
buckets = prepared.buckets;
|
||||
prepared.cache.info.scan_coverage_digest = Some(bucket_coverage_digest);
|
||||
prepared.cache
|
||||
@@ -1028,12 +992,12 @@ impl ScannerIOCache for SetDisks {
|
||||
if turn_bucket_count == 0 {
|
||||
bucket_work_complete.cancel();
|
||||
}
|
||||
for NamespaceScannerWalker {
|
||||
disk,
|
||||
mode: worker_mode,
|
||||
local_admission,
|
||||
} in walkers
|
||||
{
|
||||
let walker_count = walkers.len();
|
||||
let available_walkers = Arc::new(AvailableWalkers::new(walkers, |disk| {
|
||||
crate::storage_api::owner::EcstoreDiskAPI::endpoint(disk.as_ref()).to_string()
|
||||
}));
|
||||
for _ in 0..walker_count {
|
||||
let available_walkers = available_walkers.clone();
|
||||
let service_cohort_clone = service_cohort.clone();
|
||||
let usage_watch_cohort_clone = usage_watch_cohort.clone();
|
||||
let usage_watch_store_clone = usage_watch_store.clone();
|
||||
@@ -1065,18 +1029,18 @@ impl ScannerIOCache for SetDisks {
|
||||
let worker_retired_buckets = retired_buckets.clone();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let worker_incarnations = current_bucket_incarnations.clone();
|
||||
let remote_capabilities = match worker_mode {
|
||||
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
|
||||
NamespaceScannerWorkerMode::Coordinator => None,
|
||||
};
|
||||
#[cfg(test)]
|
||||
let worker_index = futs.len();
|
||||
futs.push(AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
#[cfg(test)]
|
||||
super::tests::observe_coordinator_bucket_worker_started(worker_index).await;
|
||||
let remote_session_id = uuid::Uuid::new_v4();
|
||||
let mut remote_session_sequence = 0_u64;
|
||||
// Returning at the top of the loop keeps the retiring disk unavailable
|
||||
// on every break, cancellation, or panic, including after requeue.
|
||||
let mut previous_walker = None;
|
||||
loop {
|
||||
if let Some(slot) = previous_walker.take() {
|
||||
available_walkers.release(slot);
|
||||
}
|
||||
// Do not prefetch a FIFO member into an independently
|
||||
// scheduled permit waiter: that can reorder admissions.
|
||||
let permit_wait_start = Instant::now();
|
||||
@@ -1089,18 +1053,39 @@ impl ScannerIOCache for SetDisks {
|
||||
break;
|
||||
}
|
||||
let permit_wait_elapsed = permit_wait_start.elapsed();
|
||||
let bucket = tokio::select! {
|
||||
let (bucket, slot) = tokio::select! {
|
||||
_ = bucket_work_complete_clone.cancelled() => break,
|
||||
_ = ctx_clone.cancelled() => break,
|
||||
bucket = async { bucket_rx_mutex_clone.lock().await.recv().await } => {
|
||||
let Some(bucket) = bucket else {
|
||||
break;
|
||||
};
|
||||
bucket
|
||||
selected = async {
|
||||
let mut receiver = bucket_rx_mutex_clone.lock().await;
|
||||
let bucket = receiver.recv().await?;
|
||||
// Select while still owning FIFO admission, so the oldest bucket
|
||||
// gets its preferred disk before later waiters can borrow it.
|
||||
let parked = crate::scanner_folder::parked_bucket_disks(&bucket.name);
|
||||
let slot = available_walkers.take_for_bucket(&bucket.name, |disk| {
|
||||
parked.iter().any(|owner| owner.ptr_eq(&Arc::downgrade(disk)))
|
||||
});
|
||||
Some((bucket, slot))
|
||||
} => match selected {
|
||||
Some(pair) => pair,
|
||||
None => break,
|
||||
}
|
||||
};
|
||||
let mut work_guard =
|
||||
BucketWorkGuard::new(remaining_bucket_work_clone.clone(), bucket_work_complete_clone.clone());
|
||||
let Some(slot) = slot else {
|
||||
// Each live queue consumer owns at most one walker. On retirement a
|
||||
// consumer and its last walker leave together.
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
break;
|
||||
};
|
||||
let slot = previous_walker.insert(slot);
|
||||
let disk = slot.walker.disk.clone();
|
||||
let local_admission = &slot.walker.local_admission;
|
||||
let remote_capabilities = match slot.walker.mode {
|
||||
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
|
||||
NamespaceScannerWorkerMode::Coordinator => None,
|
||||
};
|
||||
// Prefix hints are process-local. Never hand one to a
|
||||
// remote or legacy-coordinator disk path.
|
||||
let prefix_scan_scope = disk.is_local().then(|| scope_clone.prefix_scope_for(&bucket.name)).flatten();
|
||||
@@ -1220,8 +1205,8 @@ impl ScannerIOCache for SetDisks {
|
||||
let bucket_checkpoint_plan_digest = bucket_plans.coverage;
|
||||
|
||||
if let Some(capabilities) = remote_capabilities {
|
||||
let request_sequence = remote_session_sequence;
|
||||
let Some(next_sequence) = remote_session_sequence.checked_add(1) else {
|
||||
let request_sequence = slot.remote_session_sequence;
|
||||
let Some(next_sequence) = request_sequence.checked_add(1) else {
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
target: "rustfs::scanner::io",
|
||||
@@ -1234,7 +1219,7 @@ impl ScannerIOCache for SetDisks {
|
||||
);
|
||||
break;
|
||||
};
|
||||
remote_session_sequence = next_sequence;
|
||||
slot.remote_session_sequence = next_sequence;
|
||||
let remote_scan = crate::remote_scanner::scan_remote_bucket(
|
||||
&disk,
|
||||
if capabilities.supports_durable_progress || capabilities.supports_complete_observation || capabilities.supports_usage_watch {
|
||||
@@ -1248,7 +1233,7 @@ impl ScannerIOCache for SetDisks {
|
||||
next_cycle: want_cycle,
|
||||
leader_epoch,
|
||||
server_epoch: capabilities.server_epoch,
|
||||
session_id: remote_session_id,
|
||||
session_id: slot.remote_session_id,
|
||||
session_sequence: request_sequence,
|
||||
plans: bucket_plans,
|
||||
supports_durable_progress: capabilities.supports_durable_progress,
|
||||
@@ -1441,7 +1426,7 @@ impl ScannerIOCache for SetDisks {
|
||||
continue;
|
||||
}
|
||||
|
||||
let _local_admission = if let Some(share) = &local_admission {
|
||||
let _local_admission = if let Some(share) = local_admission {
|
||||
match share.acquire(&disk) {
|
||||
Ok(admission) => Some(admission),
|
||||
Err(e) => {
|
||||
@@ -1524,8 +1509,13 @@ impl ScannerIOCache for SetDisks {
|
||||
};
|
||||
|
||||
let mut cache = DataUsageCache::default();
|
||||
let mut revisions = match cache.load_with_revisions(store_clone_clone.clone(), &cache_name).await {
|
||||
Ok(revisions) => revisions,
|
||||
let mut revisions = match Box::pin(cache.load_for_scan(store_clone_clone.clone(), &cache_name, &scan_ctx, &bucket_budget)).await {
|
||||
Ok(Some(revisions)) => revisions,
|
||||
Ok(None) => {
|
||||
work_guard.mark_service_interrupted();
|
||||
record_partial_dirty_bucket(&partial_dirty_buckets_clone, &bucket.name).await;
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
@@ -1597,14 +1587,8 @@ impl ScannerIOCache for SetDisks {
|
||||
}
|
||||
record_completed_usage_watch(&usage_watch_completions_clone, source, &bucket.name, bucket_usage_watch.as_ref()).await;
|
||||
if let Err(e) =
|
||||
send_current_cache_root_entry(
|
||||
&bucket_result_tx_clone,
|
||||
*root,
|
||||
&cache,
|
||||
&pending_maintenance_work_clone,
|
||||
global_metrics().scan_observation(),
|
||||
)
|
||||
.await
|
||||
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
|
||||
.await
|
||||
{
|
||||
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
|
||||
error!(
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::NamespaceScannerWalker;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::sync::Mutex;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// All state belonging to one physical walker travels with it, even when a
|
||||
/// different queue consumer makes the next request on that endpoint.
|
||||
pub(super) struct WalkerSlot<T> {
|
||||
pub(super) walker: NamespaceScannerWalker<T>,
|
||||
pub(super) remote_session_id: Uuid,
|
||||
pub(super) remote_session_sequence: u64,
|
||||
endpoint: String,
|
||||
}
|
||||
|
||||
pub(super) struct AvailableWalkers<T>(Mutex<Vec<WalkerSlot<T>>>);
|
||||
|
||||
impl<T> AvailableWalkers<T> {
|
||||
pub(super) fn new(walkers: Vec<NamespaceScannerWalker<T>>, endpoint: impl Fn(&T) -> String) -> Self {
|
||||
Self(Mutex::new(
|
||||
walkers
|
||||
.into_iter()
|
||||
.map(|walker| WalkerSlot {
|
||||
endpoint: endpoint(&walker.disk),
|
||||
walker,
|
||||
remote_session_id: Uuid::new_v4(),
|
||||
remote_session_sequence: 0,
|
||||
})
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Rank available physical endpoints independently of random inventory
|
||||
/// order, waiter wakeups or the number of walker slots on a local disk.
|
||||
pub(super) fn take_for_bucket(&self, bucket: &str, parked: impl Fn(&T) -> bool) -> Option<WalkerSlot<T>> {
|
||||
let mut idle = self.0.lock().unwrap_or_else(|err| err.into_inner());
|
||||
let mut prefix = Sha256::new();
|
||||
prefix.update(bucket.as_bytes());
|
||||
prefix.update([0]);
|
||||
let index = idle
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, slot)| {
|
||||
let rank = prefix.clone().chain_update(slot.endpoint.as_bytes()).finalize();
|
||||
(parked(&slot.walker.disk), rank, slot.endpoint.as_str(), index)
|
||||
})
|
||||
.max()
|
||||
.map(|(_, _, _, index)| index)?;
|
||||
Some(idle.swap_remove(index))
|
||||
}
|
||||
|
||||
pub(super) fn release(&self, slot: WalkerSlot<T>) {
|
||||
self.0.lock().unwrap_or_else(|err| err.into_inner()).push(slot);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::scanner_io::NamespaceScannerWorkerMode;
|
||||
|
||||
fn pool(endpoints: &[&'static str]) -> AvailableWalkers<&'static str> {
|
||||
let walkers = endpoints
|
||||
.iter()
|
||||
.copied()
|
||||
.map(|disk| NamespaceScannerWalker {
|
||||
disk,
|
||||
mode: NamespaceScannerWorkerMode::Coordinator,
|
||||
local_admission: None,
|
||||
})
|
||||
.collect();
|
||||
AvailableWalkers::new(walkers, |disk| (*disk).to_owned())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_resumes_on_same_free_endpoint_despite_reordering_or_borrower_changes() {
|
||||
let first = pool(&["disk-c", "disk-a", "disk-b"]);
|
||||
let reordered = pool(&["disk-b", "disk-c", "disk-a"]);
|
||||
for bucket in ["wide", "documents", "snapshots", "videos"] {
|
||||
let mut preferred = first.take_for_bucket(bucket, |_| false).expect("preferred walker");
|
||||
let another = reordered
|
||||
.take_for_bucket(bucket, |_| false)
|
||||
.expect("reordered preferred walker");
|
||||
assert_eq!(preferred.endpoint, another.endpoint);
|
||||
let next = first
|
||||
.take_for_bucket(bucket, |_| false)
|
||||
.expect("fallback when primary is busy");
|
||||
assert_ne!(preferred.endpoint, next.endpoint);
|
||||
let alternative = next.endpoint.clone();
|
||||
let id = preferred.remote_session_id;
|
||||
preferred.remote_session_sequence += 1;
|
||||
let sequence = preferred.remote_session_sequence;
|
||||
first.release(preferred);
|
||||
let resumed = first.take_for_bucket(bucket, |_| false).expect("released primary");
|
||||
assert_eq!((resumed.remote_session_id, resumed.remote_session_sequence), (id, sequence));
|
||||
first.release(resumed);
|
||||
first.release(next);
|
||||
let parked = first
|
||||
.take_for_bucket(bucket, |disk| *disk == alternative)
|
||||
.expect("parked directory");
|
||||
assert_eq!(parked.endpoint, alternative, "follow a parked stream on a previous fallback disk");
|
||||
first.release(parked);
|
||||
reordered.release(another);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retired_slot_leaves_same_endpoint_alternative_and_does_not_leak() {
|
||||
let slots = pool(&["disk-a", "disk-a"]);
|
||||
let retiring = slots.take_for_bucket("wide", |_| false).expect("first walker");
|
||||
let survivor = slots.take_for_bucket("wide", |_| false).expect("second local walker");
|
||||
assert!(slots.take_for_bucket("wide", |_| false).is_none());
|
||||
assert_ne!(retiring.remote_session_id, survivor.remote_session_id);
|
||||
let id = survivor.remote_session_id;
|
||||
drop(retiring);
|
||||
slots.release(survivor);
|
||||
assert_eq!(
|
||||
slots
|
||||
.take_for_bucket("wide", |_| false)
|
||||
.expect("surviving walker")
|
||||
.remote_session_id,
|
||||
id
|
||||
);
|
||||
assert!(slots.take_for_bucket("wide", |_| false).is_none());
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,48 @@ struct CollectorSlotSummary<'a> {
|
||||
source_difference_sample: Option<CollectorSlotSample<'a>>,
|
||||
}
|
||||
|
||||
impl CollectorSlotSummary<'_> {
|
||||
/// Select one observed blocker. A source emitted in a different slot is valid when the set of sources matches.
|
||||
fn partial_stage(
|
||||
&self,
|
||||
bucket_plan_complete: bool,
|
||||
bucket_scan_status: ScannerBucketScanStatus,
|
||||
dirty_usage_status: DirtyUsageSnapshotStatus,
|
||||
scan_scope_matches: bool,
|
||||
candidate_rejection: Option<rustfs_scanner_metrics::usage_candidate::ScannerUsageCandidateOutcome>,
|
||||
) -> &'static str {
|
||||
if !bucket_plan_complete {
|
||||
"inventory_incomplete"
|
||||
} else if self.unobserved > 0 {
|
||||
"source_result_missing"
|
||||
} else if bucket_scan_status == ScannerBucketScanStatus::Failed {
|
||||
"bucket_worker_failed"
|
||||
} else if bucket_scan_status == ScannerBucketScanStatus::NamespaceNotFound {
|
||||
"bucket_namespace_not_found"
|
||||
} else if bucket_scan_status == ScannerBucketScanStatus::Partial {
|
||||
"bucket_walk_incomplete"
|
||||
} else if dirty_usage_status == DirtyUsageSnapshotStatus::Unverified {
|
||||
"dirty_generation_unverified"
|
||||
} else if self.missing_source > 0 {
|
||||
"missing_source"
|
||||
} else if self.cycle_mismatch > 0 {
|
||||
"source_cycle_mismatch"
|
||||
} else if self.epoch_mismatch > 0 {
|
||||
"source_epoch_mismatch"
|
||||
} else if self.missing_plan > 0 || self.plan_mismatch > 0 {
|
||||
"source_plan_mismatch"
|
||||
} else if !scan_scope_matches {
|
||||
"source_membership"
|
||||
} else if self.incomplete > 0 {
|
||||
"source_snapshot_incomplete"
|
||||
} else if self.missing_last_update > 0 {
|
||||
"missing_last_update"
|
||||
} else {
|
||||
candidate_rejection.map_or("root_coverage_incomplete", |outcome| outcome.as_label())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn summarize_collector_slots<'a>(
|
||||
slots: impl IntoIterator<Item = (usize, DataUsageCacheSource, bool, &'a DataUsageCache)>,
|
||||
cycle: u64,
|
||||
@@ -357,7 +399,6 @@ where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
|
||||
segment_reuse_activation_preflight: None,
|
||||
scope,
|
||||
remote_dirty_usage_acknowledgements: Vec::new(),
|
||||
distributed_segment_invalidation_evidence: None,
|
||||
@@ -507,7 +548,7 @@ where
|
||||
distributed,
|
||||
resolution.baseline_proof,
|
||||
);
|
||||
let scope = scoped_scan_scope_from_dirty_buckets(
|
||||
default_result(scoped_scan_scope_from_dirty_buckets(
|
||||
resolution.requested_scope,
|
||||
dirty_buckets,
|
||||
(!distributed).then_some(resolution.dirty_usage_snapshot.scopes.as_ref()),
|
||||
@@ -515,11 +556,7 @@ where
|
||||
segment_reuse_activation_preflight.scanner_segment_reuse_activated,
|
||||
resolution.all_buckets,
|
||||
resolution.baseline_proof,
|
||||
);
|
||||
ScannerBucketScopeResolutionResult {
|
||||
segment_reuse_activation_preflight: Some(segment_reuse_activation_preflight),
|
||||
..default_result(scope)
|
||||
}
|
||||
))
|
||||
}
|
||||
|
||||
/// Runs once between the bucket listing and the dirty-usage snapshot, the
|
||||
@@ -764,9 +801,6 @@ where
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if let Some(preflight) = scope_resolution.segment_reuse_activation_preflight {
|
||||
preflight.record_observation(ScannerPrefixReuseStage::ScopeResolution);
|
||||
}
|
||||
let remote_dirty_usage_acknowledgements = scope_resolution.remote_dirty_usage_acknowledgements;
|
||||
let distributed_segment_invalidation_evidence = scope_resolution.distributed_segment_invalidation_evidence;
|
||||
let scan_scope = scope_resolution.scope;
|
||||
@@ -778,14 +812,8 @@ where
|
||||
baseline_proof,
|
||||
distributed,
|
||||
);
|
||||
if let Some(fallback) = scope_fallback {
|
||||
let reason = fallback.reason;
|
||||
if let Some(reason) = scope_fallback {
|
||||
global_metrics().record_scanner_scope_fallback(reason.as_str());
|
||||
if let Some(failure) = fallback.baseline_rejection {
|
||||
global_metrics()
|
||||
.scan_observation()
|
||||
.record_baseline_rejection(failure.authoritative, failure.observed);
|
||||
}
|
||||
tracing::info!(
|
||||
target: "rustfs::scanner::io",
|
||||
event = EVENT_SCANNER_SET_STATE,
|
||||
@@ -799,7 +827,6 @@ where
|
||||
"Scanner cycle walks every bucket because its dirty-bucket scope was not admitted"
|
||||
);
|
||||
}
|
||||
let scope_fallback = scope_fallback.map(|fallback| fallback.reason);
|
||||
let segment_invalidation_proof = dirty_usage_producer_evidence(&dirty_usage_snapshot).segment_invalidation_proof();
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = resolved_scope_observer {
|
||||
@@ -812,7 +839,9 @@ where
|
||||
if all_buckets.is_empty() {
|
||||
reset_set_scan_gauges();
|
||||
if !bucket_plan_complete {
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)
|
||||
.with_partial_stage(Some("inventory_incomplete"))
|
||||
.with_publication_epoch(publication_epoch));
|
||||
}
|
||||
let publication_blocked = store.scanner_data_usage_publication_blocked().await;
|
||||
let activity_after = probe_scanner_cycle_activity(
|
||||
@@ -849,7 +878,9 @@ where
|
||||
tier_registry_generation: Some(tier_registry_generation),
|
||||
},
|
||||
) else {
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)
|
||||
.with_partial_stage(Some("empty_namespace_guard"))
|
||||
.with_publication_epoch(publication_epoch));
|
||||
};
|
||||
let (empty_usage, publication_expectation) = candidate.prepare(status, &pending_buckets);
|
||||
let usage_update_emitted = if should_publish_observational_snapshot(status) {
|
||||
@@ -858,7 +889,11 @@ where
|
||||
publish_usage_snapshot(&updates, status, &pending_buckets, empty_usage).await?
|
||||
};
|
||||
if !usage_update_emitted {
|
||||
return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch));
|
||||
let partial_stage =
|
||||
(dirty_usage_status == DirtyUsageSnapshotStatus::Unverified).then_some("dirty_generation_unverified");
|
||||
return Ok(ScannerCycleResult::new(status, None)
|
||||
.with_partial_stage(partial_stage)
|
||||
.with_publication_epoch(publication_epoch));
|
||||
}
|
||||
if status == ScannerCycleStatus::Complete {
|
||||
complete_tier_registry_cycle(want_cycle, leader_epoch);
|
||||
@@ -876,7 +911,6 @@ where
|
||||
None,
|
||||
false,
|
||||
);
|
||||
segment_reuse_activation_preflight.record_observation(ScannerPrefixReuseStage::CycleCompletion);
|
||||
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
|
||||
.with_publication_epoch(publication_epoch)
|
||||
.with_activity_digest(activity_digest)
|
||||
@@ -902,7 +936,9 @@ where
|
||||
"Scanner set state update detected missing disk sets"
|
||||
);
|
||||
reset_set_scan_gauges();
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
|
||||
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)
|
||||
.with_partial_stage(Some("source_inventory_empty"))
|
||||
.with_publication_epoch(publication_epoch));
|
||||
}
|
||||
|
||||
let dispatch_limit = scanner_max_concurrent_set_scans(total_results);
|
||||
@@ -1223,7 +1259,6 @@ where
|
||||
distributed_segment_invalidation_evidence,
|
||||
cold_zero_walk_oracle,
|
||||
);
|
||||
segment_reuse_activation_preflight.record_observation(ScannerPrefixReuseStage::CycleCompletion);
|
||||
let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results);
|
||||
let observed_cycle_floor = cache_cycle_floor.load(Ordering::Acquire);
|
||||
let required_cycle_floor = (observed_cycle_floor > want_cycle).then_some(observed_cycle_floor);
|
||||
@@ -1278,7 +1313,7 @@ where
|
||||
remote_certification_attempted,
|
||||
);
|
||||
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
|
||||
let mut completed_usage = completed_usage_candidate(
|
||||
let completed_candidate = completed_usage_candidate(
|
||||
&results,
|
||||
&ScannerSnapshotScope {
|
||||
sources: &expected_sources,
|
||||
@@ -1295,6 +1330,8 @@ where
|
||||
bucket_plan_complete,
|
||||
owner_ctx.is_cancelled(),
|
||||
);
|
||||
let candidate_rejection = completed_candidate.as_ref().err().copied();
|
||||
let mut completed_usage = completed_candidate.ok();
|
||||
if let (Some(candidate), Some(baseline)) = (
|
||||
completed_usage.as_mut(),
|
||||
persisted_usage_baseline
|
||||
@@ -1325,7 +1362,11 @@ where
|
||||
dirty_usage_status,
|
||||
activity_status,
|
||||
);
|
||||
if cycle_status == ScannerCycleStatus::Incomplete && result.is_ok() && !budget_elapsed && !owner_ctx.is_cancelled() {
|
||||
let partial_stage = if cycle_status == ScannerCycleStatus::Incomplete
|
||||
&& result.is_ok()
|
||||
&& !budget_elapsed
|
||||
&& !owner_ctx.is_cancelled()
|
||||
{
|
||||
let slots =
|
||||
set_disks
|
||||
.iter()
|
||||
@@ -1336,6 +1377,35 @@ where
|
||||
(index, DataUsageCacheSource::new(set.pool_index, set.set_index), *observed, snapshot)
|
||||
});
|
||||
let summary = summarize_collector_slots(slots, want_cycle, leader_epoch, scan_plan_digest);
|
||||
let mut stage = summary.partial_stage(
|
||||
bucket_plan_complete,
|
||||
bucket_scan_status,
|
||||
dirty_usage_status,
|
||||
scan_scope_matches,
|
||||
candidate_rejection,
|
||||
);
|
||||
// A fully traversed mixed-plan observation needs another certification sweep. Only
|
||||
// attribute the stop to that fence when every uncarried bucket has fresh source evidence.
|
||||
if stage == "bucket_walk_incomplete"
|
||||
&& let Some((snapshot, _)) = &observational_usage
|
||||
&& partial_buckets.difference(&carried_partial_buckets).all(|bucket| {
|
||||
snapshot
|
||||
.usage_snapshot_bucket_observations
|
||||
.get(bucket)
|
||||
.is_some_and(|observation| {
|
||||
observation
|
||||
.sources
|
||||
.iter()
|
||||
.all(|source| source.scanner_epoch == leader_epoch && source.scanner_cycle == want_cycle)
|
||||
&& observation
|
||||
.sources
|
||||
.iter()
|
||||
.any(|source| source.started_plan != source.requested_plan)
|
||||
})
|
||||
})
|
||||
{
|
||||
stage = "mixed_plan_recertification_pending";
|
||||
}
|
||||
let sample = summary.sample;
|
||||
let source_difference = summary.source_difference_sample;
|
||||
let sample_snapshot = sample.filter(|sample| sample.observed).map(|sample| sample.snapshot);
|
||||
@@ -1345,6 +1415,7 @@ where
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_IO,
|
||||
state = "incomplete_cycle_collector_summary",
|
||||
partial_stage = stage,
|
||||
cycle = want_cycle,
|
||||
leader_epoch,
|
||||
configured_sets = set_disks.len(),
|
||||
@@ -1390,7 +1461,10 @@ where
|
||||
sample_snapshot_plan_matches = sample_snapshot.and_then(|snapshot| snapshot.info.scan_plan_digest.map(|plan| plan == scan_plan_digest)),
|
||||
"Scanner recorded collector observations for an incomplete cycle"
|
||||
);
|
||||
}
|
||||
Some(stage)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Certification is per bucket: the pending set is read after the
|
||||
// guardrail verdict so a bucket written during the walk is reported in
|
||||
// the same publication that certifies the others. A bucket whose walk
|
||||
@@ -1583,6 +1657,7 @@ where
|
||||
complete_tier_registry_cycle(want_cycle, leader_epoch);
|
||||
}
|
||||
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
|
||||
.with_partial_stage(partial_stage)
|
||||
.with_usage_watch_proofs(usage_watch_proofs.into_values().collect())
|
||||
.with_usage_watch_retirements(usage_watch_retirements)
|
||||
.with_publication_epoch(publication_epoch)
|
||||
|
||||
@@ -15,7 +15,6 @@ use super::*;
|
||||
use crate::UNKNOWN_TIER;
|
||||
/// ScannerIODisk implementation for Disk: get_size and the per-disk bucket scan.
|
||||
use crate::data_usage_define::clone_cache_value;
|
||||
use rustfs_scanner_metrics::scan_observation::ScannerMetadataReadOutcome;
|
||||
|
||||
///
|
||||
/// Seed [`SizeSummary::tier_stats`] from the cached tier-name list.
|
||||
@@ -63,20 +62,8 @@ impl ScannerIODisk for Disk {
|
||||
|
||||
ensure_scanner_node_operation_active(cancellation.as_ref())?;
|
||||
let metadata_object_path = item.object_path();
|
||||
let metadata_result = crate::workload_admission::with_scanner_metadata_io(async {
|
||||
let read = global_metrics().start_scanner_metadata_read();
|
||||
let result = self.read_metadata(&item.bucket, &metadata_object_path).await;
|
||||
let (outcome, bytes) = match &result {
|
||||
Ok(data) => (ScannerMetadataReadOutcome::Success, u64::try_from(data.len()).unwrap_or(u64::MAX)),
|
||||
Err(e) if DiskError::is_err_object_not_found(e) || DiskError::is_err_version_not_found(e) => {
|
||||
(ScannerMetadataReadOutcome::NotFound, 0)
|
||||
}
|
||||
Err(_) => (ScannerMetadataReadOutcome::Error, 0),
|
||||
};
|
||||
read.finish(outcome, bytes);
|
||||
result
|
||||
})
|
||||
.await;
|
||||
let metadata_result =
|
||||
crate::workload_admission::with_scanner_metadata_io(self.read_metadata(&item.bucket, &metadata_object_path)).await;
|
||||
ensure_scanner_node_operation_active(cancellation.as_ref())?;
|
||||
let data = match metadata_result {
|
||||
Ok(data) => data,
|
||||
|
||||
@@ -27,10 +27,10 @@ use crate::storage_api::owner::{
|
||||
};
|
||||
use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
|
||||
use crate::{
|
||||
BUCKET_META_PREFIX, DATA_USAGE_BLOOM_NAME_PATH, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
|
||||
InstanceContext, PoolEndpoints, ScannerObjectOptions, ScannerPutObjReader, UNKNOWN_TIER,
|
||||
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
|
||||
new_disk, path2_bucket_object_with_base_path, save_config,
|
||||
DATA_USAGE_BLOOM_NAME_PATH, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints,
|
||||
ScannerObjectOptions, ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests,
|
||||
init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
|
||||
save_config,
|
||||
};
|
||||
use rustfs_concurrency::{
|
||||
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
|
||||
@@ -104,7 +104,6 @@ fn scanner_activity_preflight_defers_a_temporarily_offline_peer() {
|
||||
|
||||
#[test]
|
||||
fn scanner_segment_reuse_activation_preflight_reports_release_gate_inputs() {
|
||||
let before = global_metrics().scan_observation().snapshot();
|
||||
let preflight = scanner_segment_reuse_activation_preflight();
|
||||
|
||||
assert!(preflight.production_activation);
|
||||
@@ -125,9 +124,6 @@ fn scanner_segment_reuse_activation_preflight_reports_release_gate_inputs() {
|
||||
"distributed_without_peer_invalidation",
|
||||
]
|
||||
);
|
||||
let after = global_metrics().scan_observation().snapshot();
|
||||
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated);
|
||||
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2390,254 +2386,6 @@ async fn set_cache_send_failure_logs_receiver_and_snapshot_without_changing_retu
|
||||
assert_set_cache_context(set_cache_event(&events, "publish_failed"), Some((0, 0)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn set_snapshot_accepts_backup_catchup_to_the_same_decoded_primary() {
|
||||
use crate::data_usage_define::read_config_with_revision;
|
||||
|
||||
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
|
||||
let set = Arc::clone(&store.pools[0].disk_set[0]);
|
||||
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
|
||||
let backup_path = format!("{primary_path}.bkp");
|
||||
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
|
||||
.await
|
||||
.expect("read decoded primary bytes");
|
||||
let primary_body = primary_body.expect("primary exists");
|
||||
let mut older = cache.clone();
|
||||
older.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
|
||||
save_config(Arc::clone(&set), &backup_path, older.marshal_msg().expect("encode older backup"))
|
||||
.await
|
||||
.expect("seed a lagging backup");
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let initial = loaded
|
||||
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("capture primary with lagging backup");
|
||||
save_config(Arc::clone(&set), &backup_path, primary_body)
|
||||
.await
|
||||
.expect("backup catches up without changing primary");
|
||||
let current = loaded
|
||||
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("capture the reconciled backup");
|
||||
assert_ne!(initial.revisions(), current.revisions());
|
||||
assert!(current.permits_unchanged_baseline_since(&initial));
|
||||
|
||||
let mut candidate = cache;
|
||||
candidate.info.next_cycle += 1;
|
||||
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
|
||||
candidate.replace(
|
||||
"photos",
|
||||
DATA_USAGE_ROOT,
|
||||
DataUsageEntry {
|
||||
size: 20,
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let cycle = candidate.info.next_cycle;
|
||||
let cycle_floor = AtomicU64::new(cycle);
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
assert!(
|
||||
persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, candidate, Some(&initial), &cycle_floor, epoch)
|
||||
.await
|
||||
.is_some(),
|
||||
"content-preserving backup catchup must not veto the new snapshot"
|
||||
);
|
||||
let published = rx.try_recv().expect("new snapshot reached the collector");
|
||||
assert_eq!(published.info.next_cycle, cycle);
|
||||
assert_eq!(published.find("photos").expect("new bucket entry").size, 20);
|
||||
let (primary, primary_revision) = read_config_with_revision(Arc::clone(&set), &primary_path)
|
||||
.await
|
||||
.expect("read new primary");
|
||||
let (backup, backup_revision) = read_config_with_revision(set, &backup_path).await.expect("read new backup");
|
||||
assert_eq!(primary, backup, "backup CAS must use the reconciled current revision");
|
||||
assert_eq!(primary_revision, backup_revision);
|
||||
let persisted = DataUsageCache::unmarshal(&primary.expect("new primary body")).expect("decode new primary");
|
||||
assert_eq!(persisted.info.next_cycle, cycle);
|
||||
assert_eq!(persisted.find("photos").expect("persisted new entry").size, 20);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn set_snapshot_backup_catchup_keeps_revision_provenance_and_epoch_fences() {
|
||||
use crate::data_usage_define::read_config_with_revision;
|
||||
|
||||
for case in ["different_backup", "different_primary", "corrupt_primary", "changed_epoch"] {
|
||||
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
|
||||
let set = Arc::clone(&store.pools[0].disk_set[0]);
|
||||
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
|
||||
let backup_path = format!("{primary_path}.bkp");
|
||||
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
|
||||
.await
|
||||
.expect("read original primary");
|
||||
let primary_body = primary_body.expect("original primary exists");
|
||||
let mut older = cache.clone();
|
||||
older.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
|
||||
save_config(Arc::clone(&set), &backup_path, older.marshal_msg().expect("encode lagging backup"))
|
||||
.await
|
||||
.expect("seed lagging backup");
|
||||
if case == "corrupt_primary" {
|
||||
save_config(Arc::clone(&set), &primary_path, b"corrupt-primary".to_vec())
|
||||
.await
|
||||
.expect("inject primary corruption before the initial load");
|
||||
}
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let initial = loaded
|
||||
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("capture original baseline source and revisions");
|
||||
let mut replacement = cache.clone();
|
||||
replacement.info.scan_execution_digest = Some(DataUsageScanPlanDigest([9; 32]));
|
||||
match case {
|
||||
"different_backup" => {
|
||||
save_config(Arc::clone(&set), &backup_path, replacement.marshal_msg().expect("encode changed backup"))
|
||||
.await
|
||||
.expect("backup changes to different contents");
|
||||
}
|
||||
"different_primary" => {
|
||||
let body = replacement.marshal_msg().expect("encode changed primary");
|
||||
save_config(Arc::clone(&set), &primary_path, body.clone())
|
||||
.await
|
||||
.expect("primary advances");
|
||||
save_config(Arc::clone(&set), &backup_path, body)
|
||||
.await
|
||||
.expect("backup follows changed primary");
|
||||
}
|
||||
"corrupt_primary" => {
|
||||
save_config(Arc::clone(&set), &backup_path, b"corrupt-primary".to_vec())
|
||||
.await
|
||||
.expect("backup matches the corrupt primary ETag without a decoded primary proof");
|
||||
}
|
||||
"changed_epoch" => {
|
||||
save_config(Arc::clone(&set), &backup_path, primary_body)
|
||||
.await
|
||||
.expect("backup catches up before epoch refusal");
|
||||
}
|
||||
_ => unreachable!("fixed test cases"),
|
||||
}
|
||||
let before_primary = read_config_with_revision(Arc::clone(&set), &primary_path)
|
||||
.await
|
||||
.expect("current primary");
|
||||
let before_backup = read_config_with_revision(Arc::clone(&set), &backup_path)
|
||||
.await
|
||||
.expect("current backup");
|
||||
let mut candidate = cache;
|
||||
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
|
||||
let cycle_floor = AtomicU64::new(candidate.info.next_cycle);
|
||||
let requested_epoch = if case == "changed_epoch" {
|
||||
epoch.checked_add(1).expect("fixture epoch can advance")
|
||||
} else {
|
||||
epoch
|
||||
};
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
let (result, events) = capture_set_cache_logs(persist_and_publish_cache_snapshot(
|
||||
Arc::clone(&set),
|
||||
&tx,
|
||||
candidate,
|
||||
Some(&initial),
|
||||
&cycle_floor,
|
||||
requested_epoch,
|
||||
))
|
||||
.await;
|
||||
assert!(result.is_none(), "{case} must still refuse publication");
|
||||
assert!(
|
||||
matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
|
||||
"{case} forwards no snapshot"
|
||||
);
|
||||
let expected_state = if case == "changed_epoch" {
|
||||
"save_failed"
|
||||
} else {
|
||||
"scan_baseline_revision_changed"
|
||||
};
|
||||
assert_set_cache_context(set_cache_event(&events, expected_state), Some((0, 0)));
|
||||
assert_eq!(
|
||||
read_config_with_revision(Arc::clone(&set), &primary_path)
|
||||
.await
|
||||
.expect("protected primary"),
|
||||
before_primary,
|
||||
"{case} does not overwrite primary"
|
||||
);
|
||||
assert_eq!(
|
||||
read_config_with_revision(set, &backup_path).await.expect("protected backup"),
|
||||
before_backup,
|
||||
"{case} does not overwrite backup"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn set_cache_cas_preserves_updates_after_backup_catchup_validation() {
|
||||
use crate::data_usage_define::read_config_with_revision;
|
||||
|
||||
for changed_copy in ["primary", "backup"] {
|
||||
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
|
||||
let set = Arc::clone(&store.pools[0].disk_set[0]);
|
||||
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
|
||||
let backup_path = format!("{primary_path}.bkp");
|
||||
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
|
||||
.await
|
||||
.expect("read initial primary");
|
||||
let mut lagging = cache.clone();
|
||||
lagging.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
|
||||
save_config(Arc::clone(&set), &backup_path, lagging.marshal_msg().expect("lagging backup bytes"))
|
||||
.await
|
||||
.expect("seed backup lag");
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let initial = loaded
|
||||
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("read initial baseline");
|
||||
save_config(Arc::clone(&set), &backup_path, primary_body.expect("initial primary exists"))
|
||||
.await
|
||||
.expect("finish backup catchup");
|
||||
let current = loaded
|
||||
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("read reconciled baseline");
|
||||
assert!(current.permits_unchanged_baseline_since(&initial));
|
||||
let changed_path = if changed_copy == "primary" {
|
||||
&primary_path
|
||||
} else {
|
||||
&backup_path
|
||||
};
|
||||
let mut concurrent = cache.clone();
|
||||
concurrent.info.scan_execution_digest = Some(DataUsageScanPlanDigest([9; 32]));
|
||||
let concurrent_body = concurrent.marshal_msg().expect("encode concurrent update");
|
||||
save_config(Arc::clone(&set), changed_path, concurrent_body.clone())
|
||||
.await
|
||||
.expect("copy changes after the baseline validation");
|
||||
let mut candidate = cache;
|
||||
candidate.info.next_cycle += 1;
|
||||
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
|
||||
let result = candidate
|
||||
.save_with_revisions_for_epoch(Arc::clone(&set), DATA_USAGE_CACHE_NAME, current.revisions(), epoch)
|
||||
.await;
|
||||
if changed_copy == "primary" {
|
||||
assert!(result.is_err(), "primary CAS refuses the intervening update");
|
||||
} else {
|
||||
result.expect("primary remains durable when the best-effort backup CAS refuses an update");
|
||||
}
|
||||
let (protected, _) = read_config_with_revision(Arc::clone(&set), changed_path)
|
||||
.await
|
||||
.expect("read protected concurrent copy");
|
||||
assert_eq!(
|
||||
protected,
|
||||
Some(concurrent_body),
|
||||
"{changed_copy} CAS must not overwrite the concurrent update"
|
||||
);
|
||||
let (primary, _) = read_config_with_revision(set, &primary_path)
|
||||
.await
|
||||
.expect("read resulting primary");
|
||||
let primary = DataUsageCache::unmarshal(&primary.expect("primary exists")).expect("primary remains decodable");
|
||||
assert_eq!(
|
||||
primary.info.scan_execution_digest,
|
||||
Some(DataUsageScanPlanDigest([if changed_copy == "primary" { 9 } else { 3 }; 32])),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers() {
|
||||
@@ -2652,7 +2400,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
|
||||
.expect("seed legacy set cache");
|
||||
let mut persisted = DataUsageCache::default();
|
||||
let initial = persisted
|
||||
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
|
||||
.load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("capture the shared starting revision");
|
||||
let mut fresh = legacy.clone();
|
||||
@@ -2678,7 +2426,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
|
||||
assert_eq!(published.find("photos").expect("published bucket").size, 20);
|
||||
assert_eq!(published.info.scan_execution_digest, fresh.info.scan_execution_digest);
|
||||
let current = persisted
|
||||
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
|
||||
.load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("capture the current revision for the unidentified execution");
|
||||
|
||||
@@ -2969,130 +2717,6 @@ fn scoped_scan_requires_a_complete_authoritative_baseline_with_exact_set_provena
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_scan_baseline_rejections_preserve_the_existing_first_failed_check() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let expected_sources = HashSet::from([source]);
|
||||
let digest = DataUsageScanPlanDigest([9; 32]);
|
||||
let baseline = complete_usage_baseline(source, digest, 7, 11);
|
||||
let snapshot: DataUsageInfo = serde_json::from_slice(&baseline).expect("decode complete baseline");
|
||||
let proof = ScannerCacheBaselineProof {
|
||||
authoritative_data: Some(&baseline),
|
||||
observed_candidate_data: None,
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest: digest,
|
||||
};
|
||||
type RejectionCase = (ScannerBaselineRejection, fn(&mut DataUsageInfo));
|
||||
let cases: [RejectionCase; 13] = [
|
||||
(ScannerBaselineRejection::Incomplete, |s| s.usage_snapshot_complete = false),
|
||||
(ScannerBaselineRejection::Partial, |s| s.usage_snapshot_partial = true),
|
||||
(ScannerBaselineRejection::Marks, |s| s.usage_snapshot_converged = None),
|
||||
(ScannerBaselineRejection::Epoch, |s| s.scanner_epoch = Some(10)),
|
||||
(ScannerBaselineRejection::SourceCount, |s| s.usage_snapshot_set_states.clear()),
|
||||
(ScannerBaselineRejection::PlanMissing, |s| {
|
||||
s.usage_snapshot_set_states[0].scan_plan_digest = None
|
||||
}),
|
||||
(ScannerBaselineRejection::PlanMismatch, |s| {
|
||||
s.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32])
|
||||
}),
|
||||
(ScannerBaselineRejection::SourceMembership, |s| {
|
||||
s.usage_snapshot_set_states[0].set_index = 3
|
||||
}),
|
||||
(ScannerBaselineRejection::SetIncomplete, |s| {
|
||||
s.usage_snapshot_set_states[0].complete = false
|
||||
}),
|
||||
(ScannerBaselineRejection::SetTombstone, |s| {
|
||||
s.usage_snapshot_set_states[0].tombstone = true
|
||||
}),
|
||||
(ScannerBaselineRejection::SetEpoch, |s| {
|
||||
s.usage_snapshot_set_states[0].scanner_epoch = None
|
||||
}),
|
||||
(ScannerBaselineRejection::SetCycle, |s| {
|
||||
s.usage_snapshot_set_states[0].scanner_cycle = Some(9)
|
||||
}),
|
||||
(ScannerBaselineRejection::SetCycle, |s| {
|
||||
s.usage_snapshot_set_states[0].scanner_cycle = None
|
||||
}),
|
||||
];
|
||||
assert_eq!(validate_scanner_cache_snapshot_plan_digest(&snapshot, proof, None), Ok(digest));
|
||||
for (expected, mutate) in cases {
|
||||
let mut invalid = snapshot.clone();
|
||||
mutate(&mut invalid);
|
||||
assert_eq!(validate_scanner_cache_snapshot_plan_digest(&invalid, proof, None), Err(expected));
|
||||
assert_eq!(complete_scanner_cache_snapshot_plan_digest(&invalid, proof, None), None);
|
||||
}
|
||||
let mut invalid = snapshot.clone();
|
||||
invalid.usage_snapshot_partial = true;
|
||||
invalid.scanner_epoch = Some(10);
|
||||
assert_eq!(
|
||||
validate_scanner_cache_snapshot_plan_digest(&invalid, proof, None),
|
||||
Err(ScannerBaselineRejection::Partial)
|
||||
);
|
||||
|
||||
let two_sources = HashSet::from([source, DataUsageCacheSource::new(1, 3)]);
|
||||
let mut two_sets = snapshot.clone();
|
||||
two_sets
|
||||
.usage_snapshot_set_states
|
||||
.push(two_sets.usage_snapshot_set_states[0].clone());
|
||||
let two_set_proof = ScannerCacheBaselineProof {
|
||||
expected_sources: &two_sources,
|
||||
..proof
|
||||
};
|
||||
assert_eq!(
|
||||
validate_scanner_cache_snapshot_plan_digest(&two_sets, two_set_proof, None),
|
||||
Err(ScannerBaselineRejection::SourceDuplicate)
|
||||
);
|
||||
two_sets.usage_snapshot_set_states[1].set_index = 3;
|
||||
two_sets.usage_snapshot_set_states[1].scan_plan_digest = Some([8; 32]);
|
||||
assert_eq!(
|
||||
validate_scanner_cache_snapshot_plan_digest(&two_sets, two_set_proof, None),
|
||||
Err(ScannerBaselineRejection::SetPlan)
|
||||
);
|
||||
|
||||
let undecodable = Bytes::from_static(b"not-json");
|
||||
for (data, expected) in [
|
||||
(None, ScannerBaselineRejection::Missing),
|
||||
(Some(&undecodable), ScannerBaselineRejection::Decode),
|
||||
] {
|
||||
assert_eq!(
|
||||
validate_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
|
||||
authoritative_data: data,
|
||||
..proof
|
||||
}),
|
||||
Err(ScannerBaselineValidationFailure {
|
||||
authoritative: expected,
|
||||
observed: None
|
||||
})
|
||||
);
|
||||
}
|
||||
let mut wrong_epoch = snapshot;
|
||||
wrong_epoch.scanner_epoch = Some(10);
|
||||
let wrong_epoch = Bytes::from(serde_json::to_vec(&wrong_epoch).expect("encode stale epoch baseline"));
|
||||
let stale_proof = ScannerCacheBaselineProof {
|
||||
authoritative_data: Some(&wrong_epoch),
|
||||
..proof
|
||||
};
|
||||
assert_eq!(
|
||||
validate_scanner_cache_baseline_plan_digest(stale_proof),
|
||||
Err(ScannerBaselineValidationFailure {
|
||||
authoritative: ScannerBaselineRejection::Epoch,
|
||||
observed: Some(ScannerBaselineRejection::Missing),
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
validate_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
|
||||
observed_candidate_data: Some(&undecodable),
|
||||
..stale_proof
|
||||
}),
|
||||
Err(ScannerBaselineValidationFailure {
|
||||
authoritative: ScannerBaselineRejection::Epoch,
|
||||
observed: Some(ScannerBaselineRejection::Decode)
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_scan_accepts_only_a_complete_observation_tied_to_the_authoritative_baseline() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
@@ -3526,12 +3150,6 @@ fn distributed_segment_reuse_activation_keeps_remote_dirty_buckets_at_bucket_sco
|
||||
result.scope.prefix_scope_for("photos").is_some(),
|
||||
"durable local producer proof may activate local segment reuse after distributed ACK capability evidence"
|
||||
);
|
||||
assert!(
|
||||
result
|
||||
.segment_reuse_activation_preflight
|
||||
.expect("actual scope proof is retained for observation")
|
||||
.scanner_segment_reuse_activated
|
||||
);
|
||||
assert!(
|
||||
result.scope.prefix_scope_for("archive").is_none(),
|
||||
"remote dirty usage has only bucket-granularity evidence and must not be narrowed to a local prefix"
|
||||
@@ -3950,10 +3568,6 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
|
||||
.checked_flatten("stable")
|
||||
.expect("selected bucket placeholder should exist");
|
||||
assert_eq!((stable.size, stable.objects), (0, 0));
|
||||
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
|
||||
prepared.record_cold_bucket_reuse(&observation);
|
||||
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 0);
|
||||
|
||||
assert!(prepared.cache.find("stable/prefix").is_none());
|
||||
assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0)));
|
||||
assert!(prepared.cache.find("deleted").is_none());
|
||||
@@ -3965,124 +3579,6 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
|
||||
assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cold_bucket_reuse_observation_rejects_unbound_or_failed_preparation() {
|
||||
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
|
||||
let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20)], baseline_digest);
|
||||
old_cache.info.scan_bucket_incarnations = test_bucket_incarnations(&["stable", "dirty"]);
|
||||
let incarnations = old_cache.info.scan_bucket_incarnations.clone();
|
||||
let buckets = vec![
|
||||
bucket_info_with_created_time("stable"),
|
||||
bucket_info_with_created_time("dirty"),
|
||||
];
|
||||
let scope = ScannerBucketScanScope {
|
||||
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
|
||||
selected_bucket_prefixes: None,
|
||||
baseline_scan_plan_digest: Some(baseline_digest),
|
||||
};
|
||||
let generation = ScannerSetCacheGeneration {
|
||||
want_cycle: 8,
|
||||
leader_epoch: 11,
|
||||
tier_registry_generation: 13,
|
||||
source: DataUsageCacheSource::new(1, 2),
|
||||
scan_plan_digest: DataUsageScanPlanDigest([2; 32]),
|
||||
};
|
||||
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
|
||||
let prepared = prepare_scoped_set_scan(&old_cache, &buckets, &buckets, &scope, generation, Some(&incarnations))
|
||||
.expect("bound complete cache permits a cold bucket copy");
|
||||
prepared.record_cold_bucket_reuse(&observation);
|
||||
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 1);
|
||||
|
||||
for case in ["incomplete", "duplicate", "nil_incarnation"] {
|
||||
let mut invalid_cache = old_cache.clone();
|
||||
let mut invalid_buckets = buckets.clone();
|
||||
let mut invalid_incarnations = incarnations.clone();
|
||||
match case {
|
||||
"incomplete" => invalid_cache.info.snapshot_complete = false,
|
||||
"duplicate" => invalid_buckets.push(buckets[0].clone()),
|
||||
"nil_incarnation" => {
|
||||
invalid_incarnations.insert("stable".to_string(), Uuid::nil());
|
||||
}
|
||||
_ => unreachable!("fixed test cases"),
|
||||
}
|
||||
let prepared = prepare_scoped_set_scan(
|
||||
&invalid_cache,
|
||||
&invalid_buckets,
|
||||
&invalid_buckets,
|
||||
&scope,
|
||||
generation,
|
||||
Some(&invalid_incarnations),
|
||||
);
|
||||
assert!(prepared.is_none(), "{case} must not return a reusable prepared cache");
|
||||
if let Some(prepared) = prepared {
|
||||
prepared.record_cold_bucket_reuse(&observation);
|
||||
}
|
||||
assert_eq!(
|
||||
observation.snapshot().bucket_reuse["scoped_cold_bucket"].count,
|
||||
1,
|
||||
"{case} does not count completed reuse"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_root_reuse_counts_only_successful_collector_delivery() {
|
||||
use rustfs_scanner_metrics::scan_observation::ScannerScanObservation;
|
||||
|
||||
let observation = ScannerScanObservation::default();
|
||||
let cache = DataUsageCache::default();
|
||||
let pending = AtomicBool::new(false);
|
||||
let root = || DataUsageEntryInfo {
|
||||
name: "bucket".to_string(),
|
||||
parent: DATA_USAGE_ROOT.to_string(),
|
||||
entry: DataUsageEntry {
|
||||
size: 10,
|
||||
objects: 2,
|
||||
..Default::default()
|
||||
},
|
||||
bucket_incarnation: Some(Uuid::from_u128(7)),
|
||||
tier_registry_generation: None,
|
||||
value_origin: None,
|
||||
};
|
||||
let (sender, mut receiver) = mpsc::channel(1);
|
||||
super::io_cache::send_current_cache_root_entry(&sender, root(), &cache, &pending, &observation)
|
||||
.await
|
||||
.expect("current root is delivered to the collector");
|
||||
let delivered = receiver.recv().await.expect("collector received the root");
|
||||
assert_eq!((delivered.entry.size, delivered.entry.objects), (10, 2));
|
||||
assert_eq!(observation.snapshot().bucket_reuse["current_root"].count, 1);
|
||||
drop(receiver);
|
||||
assert!(
|
||||
super::io_cache::send_current_cache_root_entry(&sender, root(), &cache, &pending, &observation)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
observation.snapshot().bucket_reuse["current_root"].count,
|
||||
1,
|
||||
"closed collector is not completed reuse"
|
||||
);
|
||||
|
||||
let (sender, _receiver) = mpsc::channel(1);
|
||||
sender.send(root()).await.expect("fill bounded collector channel");
|
||||
{
|
||||
let mut send = Box::pin(super::io_cache::send_current_cache_root_entry(
|
||||
&sender,
|
||||
root(),
|
||||
&cache,
|
||||
&pending,
|
||||
&observation,
|
||||
));
|
||||
assert!(futures::poll!(send.as_mut()).is_pending(), "full collector keeps reuse unfinished");
|
||||
}
|
||||
assert_eq!(
|
||||
observation.snapshot().bucket_reuse["current_root"].count,
|
||||
1,
|
||||
"dropping unfinished delivery is not reuse"
|
||||
);
|
||||
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() {
|
||||
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
|
||||
@@ -4124,12 +3620,6 @@ fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() {
|
||||
)
|
||||
.expect("matching bucket incarnations should authorize cold bucket reuse");
|
||||
|
||||
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
|
||||
prepared.record_cold_bucket_reuse(&observation);
|
||||
let observed = observation.snapshot();
|
||||
assert_eq!(observed.bucket_reuse["scoped_cold_bucket"].count, 1);
|
||||
assert_eq!(observed.bucket_reuse["current_root"].count, 0);
|
||||
assert!(observed.cache_reuse.values().all(|reuse| reuse.count == 0));
|
||||
assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(), ["dirty"]);
|
||||
let stable = prepared
|
||||
.cache
|
||||
@@ -5135,36 +4625,12 @@ async fn get_size_treats_missing_metadata_as_skip_file() {
|
||||
debug: false,
|
||||
};
|
||||
|
||||
let before = global_metrics().scan_observation().snapshot().metadata_reads;
|
||||
let err = disk
|
||||
.get_size(item.clone())
|
||||
.get_size(item)
|
||||
.await
|
||||
.expect_err("missing metadata should be skipped instead of reported as a scanner failure");
|
||||
assert!(matches!(err, StorageError::Io(ref io) if io.to_string() == SCANNER_SKIP_FILE_ERROR));
|
||||
|
||||
let missing = global_metrics().scan_observation().snapshot().metadata_reads;
|
||||
assert_eq!(missing.attempts, before.attempts + 1);
|
||||
assert_eq!(missing.not_found, before.not_found + 1);
|
||||
assert_eq!(missing.bytes, before.bytes);
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
tokio::fs::create_dir_all(&object_dir)
|
||||
.await
|
||||
.expect("recreate object directory");
|
||||
std::os::unix::fs::symlink(STORAGE_FORMAT_FILE, &metadata_path).expect("inject a metadata symlink loop");
|
||||
let err = disk
|
||||
.get_size(item)
|
||||
.await
|
||||
.expect_err("symlink loop must remain a read failure");
|
||||
assert!(is_scanner_metadata_transient_error(&err), "read error: {err:?}");
|
||||
let failed = global_metrics().scan_observation().snapshot().metadata_reads;
|
||||
assert_eq!(failed.attempts, missing.attempts + 1);
|
||||
assert_eq!(failed.errors, missing.errors + 1);
|
||||
assert_eq!(failed.succeeded, before.succeeded);
|
||||
assert_eq!(failed.bytes, before.bytes);
|
||||
}
|
||||
|
||||
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
|
||||
}
|
||||
|
||||
@@ -5215,7 +4681,6 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
|
||||
debug: false,
|
||||
};
|
||||
|
||||
let before = global_metrics().scan_observation().snapshot().metadata_reads;
|
||||
for contents in [b"".as_slice(), b"not-valid-filemeta".as_slice()] {
|
||||
tokio::fs::write(&metadata_path, contents)
|
||||
.await
|
||||
@@ -5227,16 +4692,6 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
|
||||
assert!(is_scanner_metadata_corrupt_error(&err));
|
||||
}
|
||||
|
||||
let reads = global_metrics().scan_observation().snapshot().metadata_reads;
|
||||
assert_eq!(reads.attempts, before.attempts + 2);
|
||||
assert_eq!(
|
||||
reads.succeeded,
|
||||
before.succeeded + 2,
|
||||
"storage reads succeed before decoding rejects corruption"
|
||||
);
|
||||
assert_eq!(reads.errors, before.errors);
|
||||
assert_eq!(reads.bytes, before.bytes + 18);
|
||||
|
||||
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
|
||||
}
|
||||
|
||||
@@ -5309,17 +4764,11 @@ async fn get_size_counts_delete_markers_separately_from_versions() {
|
||||
debug: false,
|
||||
};
|
||||
|
||||
let before = global_metrics().scan_observation().snapshot().metadata_reads;
|
||||
let expected_bytes = tokio::fs::metadata(&metadata_path).await.expect("stat stored metadata").len();
|
||||
let summary = disk.get_size(item).await.expect("scanner should read versioned metadata");
|
||||
|
||||
assert_eq!(summary.versions, 2);
|
||||
assert_eq!(summary.delete_markers, 1);
|
||||
assert_eq!(summary.total_size, 30);
|
||||
let reads = global_metrics().scan_observation().snapshot().metadata_reads;
|
||||
assert_eq!(reads.attempts, before.attempts + 1);
|
||||
assert_eq!(reads.succeeded, before.succeeded + 1);
|
||||
assert_eq!(reads.bytes, before.bytes + expected_bytes);
|
||||
|
||||
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
|
||||
}
|
||||
|
||||
@@ -127,6 +127,8 @@ async fn collector_reports_an_unobserved_slot_after_a_stale_set_snapshot() {
|
||||
let events = collector_events(&events);
|
||||
assert_eq!(events.len(), 1, "a non-budget incomplete cycle needs one collector observation");
|
||||
let fields = events[0];
|
||||
assert_eq!(fields["partial_stage"], "source_result_missing");
|
||||
assert_eq!(result.partial_stage, Some("source_result_missing"));
|
||||
assert_eq!(fields["cycle"], 7);
|
||||
assert_eq!(fields["leader_epoch"], LEADER_EPOCH);
|
||||
assert_eq!(fields["expected_sources"], 1);
|
||||
@@ -171,6 +173,7 @@ async fn collector_reports_received_incomplete_cache_only_for_non_budget_partial
|
||||
let events = collector_events(&events);
|
||||
assert_eq!(events.len(), 1, "a non-budget incomplete cycle needs one collector observation");
|
||||
let fields = events[0];
|
||||
assert_eq!(fields["partial_stage"], second.partial_stage.expect("non-budget incomplete stage"));
|
||||
assert_eq!(fields["observed_slots"], 1);
|
||||
assert_eq!(fields["unobserved_slots"], 0);
|
||||
assert_eq!(fields["sample_observed"], true);
|
||||
@@ -187,6 +190,7 @@ async fn collector_reports_received_incomplete_cache_only_for_non_budget_partial
|
||||
|
||||
let ((third, snapshot, _), events) = capture_cycle(run_cycle(&store, 3, None, false)).await;
|
||||
assert_eq!(third.status, ScannerCycleStatus::Complete);
|
||||
assert_eq!(third.partial_stage, None);
|
||||
assert!(snapshot.is_some_and(|snapshot| snapshot.usage_snapshot_complete));
|
||||
assert!(
|
||||
collector_events(&events).is_empty(),
|
||||
|
||||
@@ -183,11 +183,9 @@ async fn dirty_entry_removed_during_listing_still_forces_a_full_scan() {
|
||||
);
|
||||
assert_eq!(second.buckets_usage["hot-bucket"].objects_count, 2);
|
||||
assert_eq!(second.buckets_usage["cold-bucket"].objects_count, 2);
|
||||
let report = global_metrics().report().await;
|
||||
let freshness = report.usage_freshness;
|
||||
let freshness = global_metrics().report().await.usage_freshness;
|
||||
assert_eq!(freshness.scope_fallback_total, fallbacks_before + 1);
|
||||
assert_eq!(freshness.last_scope_fallback_reason, "dirty_snapshot_incomplete");
|
||||
assert_eq!(report.scan_observation.scope_fallbacks["dirty_snapshot_incomplete"], 1);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
@@ -201,53 +199,11 @@ async fn required_full_scan_is_not_reported_as_a_scope_fallback() {
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
super::io_cycle::set_bucket_listing_window_hook_for_tests(|| record_dirty_usage_bucket("created-after-listing"));
|
||||
|
||||
let before = global_metrics().scan_observation().snapshot();
|
||||
let (result, _, scope) = run_cycle(&store, 2, Some(baseline), true).await;
|
||||
|
||||
assert!(scope.is_default());
|
||||
assert_eq!(result.pending_dirty_usage_buckets(), 1);
|
||||
assert_eq!(result.scope_fallback(), None, "scheduled maintenance walks are not degradations");
|
||||
assert_eq!(result.completed_scope_fallback(), None);
|
||||
let after = global_metrics().scan_observation().snapshot();
|
||||
assert_eq!(after.scope_fallbacks, before.scope_fallbacks);
|
||||
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated);
|
||||
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated + 1);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn baseline_unavailable_cycle_records_real_scope_and_baseline_rejections_once() {
|
||||
let (_temp_dir, store) = setup_local_scanner_store(1, 1).await;
|
||||
let _baseline = converged_baseline(&store).await;
|
||||
put_fixture_object(&store, "hot-bucket", "second").await;
|
||||
record_dirty_usage_bucket("hot-bucket");
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
let before = global_metrics().scan_observation().snapshot();
|
||||
let (result, snapshot, scope) = run_cycle(&store, 2, Some(Bytes::from_static(b"not-json")), false).await;
|
||||
assert!(scope.is_default());
|
||||
assert_eq!(result.status, ScannerCycleStatus::Complete);
|
||||
assert_eq!(result.scope_fallback(), Some(ScannerScopeFallbackReason::BaselineUnavailable));
|
||||
assert_eq!(snapshot.buckets_usage["hot-bucket"].objects_count, 2);
|
||||
let after = global_metrics().scan_observation().snapshot();
|
||||
assert_eq!(
|
||||
after.scope_fallbacks["baseline_unavailable"],
|
||||
before.scope_fallbacks["baseline_unavailable"] + 1
|
||||
);
|
||||
assert_eq!(
|
||||
after.authoritative_baseline_rejections["decode"],
|
||||
before.authoritative_baseline_rejections["decode"] + 1
|
||||
);
|
||||
assert_eq!(
|
||||
after.observed_baseline_rejections, before.observed_baseline_rejections,
|
||||
"an undecodable authority never admits an observation"
|
||||
);
|
||||
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated + 1);
|
||||
assert_eq!(after.prefix_scope_resolution.activated, before.prefix_scope_resolution.activated);
|
||||
assert_eq!(
|
||||
after.prefix_scope_resolution.blockers["missing_cold_zero_walk_oracle"],
|
||||
before.prefix_scope_resolution.blockers["missing_cold_zero_walk_oracle"] + 1
|
||||
);
|
||||
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated + 1);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! A bucket whose walk stops early can publish its last certified value as
|
||||
//! pending. This fixture tests usage-scan status, not the outer scanner cycle
|
||||
//! outcome: budget exhaustion may leave complete validated usage while the
|
||||
//! coordinator records `Partial`. A continuation under a newer plan exercises
|
||||
//! `mixed_coverage` (rustfs/rustfs#8494). A bucket that never certified keeps
|
||||
//! the usage scan incomplete.
|
||||
//! A bucket whose walk stops early without exhausting the cycle budget (here
|
||||
//! a sweep resumed under a newer plan, the `mixed_coverage` case of
|
||||
//! rustfs/rustfs#8494) no longer makes the whole cycle `Incomplete`. The set
|
||||
//! publishes the bucket's last certified value, the cycle completes with the
|
||||
//! bucket pending, and the next cycle walks it again. A bucket that never
|
||||
//! certified keeps the cycle incomplete.
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -38,7 +38,8 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
// Real checkpoint commits need the runtime drain grace after the count budget stops the walk.
|
||||
// These cycles persist real bucket checkpoints and source caches. Keep
|
||||
// CI storage latency inside the production drain window after a count stop.
|
||||
budget.use_production_drain_grace_for_tests();
|
||||
let (updates, mut receiver) = mpsc::channel(1);
|
||||
let result = tokio::time::timeout(
|
||||
@@ -48,7 +49,7 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
|
||||
ScannerCycleRequest {
|
||||
owner_ctx: ctx.clone(),
|
||||
ctx: budget.token(),
|
||||
budget: Arc::clone(&budget),
|
||||
budget,
|
||||
updates,
|
||||
want_cycle: cycle,
|
||||
leader_epoch: LEADER_EPOCH,
|
||||
@@ -68,13 +69,6 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
|
||||
.await
|
||||
.expect("cycle should finish within the test deadline")
|
||||
.expect("cycle should succeed");
|
||||
if options.max_objects.is_some() {
|
||||
assert_eq!(
|
||||
budget.reason(),
|
||||
Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects),
|
||||
"a count-limited fixture must stop for its object budget"
|
||||
);
|
||||
}
|
||||
let snapshot = receiver.recv().await;
|
||||
assert!(receiver.recv().await.is_none(), "a cycle publishes at most one terminal snapshot");
|
||||
(result, snapshot)
|
||||
@@ -131,8 +125,9 @@ async fn partial_bucket_with_a_certified_value_completes_the_cycle_as_pending()
|
||||
assert!(result.clear_verified_usage().is_empty());
|
||||
assert!(dirty_usage_buckets_for_tests().is_empty());
|
||||
|
||||
// Cycle 3: the budget stops the walk and saves a checkpoint. Usage can
|
||||
// carry the certified value as pending; the outer coordinator records Partial.
|
||||
// Cycle 3: the object budget stops the hot bucket's walk and leaves a
|
||||
// checkpoint under this cycle's plan. Its certified value is carried, so
|
||||
// budget exhaustion leaves the bucket pending without blocking the cycle.
|
||||
for object in ["p4/third", "p5/third"] {
|
||||
put_fixture_object(&store, "hot-bucket", object).await;
|
||||
}
|
||||
@@ -149,38 +144,12 @@ async fn partial_bucket_with_a_certified_value_completes_the_cycle_as_pending()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
result.status,
|
||||
ScannerCycleStatus::Complete,
|
||||
"certified carried usage remains complete after a budget stop"
|
||||
);
|
||||
let third = third.expect("validated carried usage publishes after the budget stop");
|
||||
let third = third.expect("a carried partial cycle publishes");
|
||||
assert_eq!(result.status, ScannerCycleStatus::Complete);
|
||||
assert_eq!(result.pending_dirty_usage_buckets(), 1);
|
||||
assert!(!third.usage_snapshot_observational);
|
||||
assert_eq!(third.usage_snapshot_converged, Some(false));
|
||||
assert_eq!(third.usage_snapshot_pending_buckets, vec!["hot-bucket".to_string()]);
|
||||
assert_eq!(third.usage_snapshot_pending_bucket_count, 1);
|
||||
assert!(third.is_complete_bucket_usage_snapshot());
|
||||
let carried_hot = &third.buckets_usage["hot-bucket"];
|
||||
let certified_hot = &second.buckets_usage["hot-bucket"];
|
||||
assert_eq!(carried_hot.usage_value_state, Some(rustfs_data_usage::BucketUsageValueState::Carried));
|
||||
assert!(certified_hot.usage_certification.is_some(), "cycle 2 certified the prior hot value");
|
||||
assert_eq!(
|
||||
carried_hot.usage_certification, certified_hot.usage_certification,
|
||||
"carrying must not mint a new certificate"
|
||||
);
|
||||
assert_eq!(carried_hot.objects_count, 4);
|
||||
assert_eq!(carried_hot.size, certified_hot.size);
|
||||
assert_eq!(third.buckets_usage["cold-bucket"].objects_count, 1);
|
||||
assert_eq!(third.buckets_usage["idle-bucket"].objects_count, 1);
|
||||
assert_eq!(third.objects_total_count, 6);
|
||||
// Only hot was dirty in this scoped scan; it received no fresh certification or ACK.
|
||||
assert!(result.clear_verified_usage().is_empty());
|
||||
assert_eq!(
|
||||
dirty_usage_buckets_for_tests().keys().cloned().collect::<Vec<_>>(),
|
||||
vec!["hot-bucket".to_string()],
|
||||
"the budget-stopped hot bucket retains its dirty responsibility"
|
||||
);
|
||||
assert_eq!(third.buckets_usage["hot-bucket"].objects_count, 4);
|
||||
let set = Arc::clone(&store.pools[0].disk_set[0]);
|
||||
let mut interrupted = DataUsageCache::default();
|
||||
interrupted
|
||||
|
||||
@@ -82,7 +82,9 @@ fn completed_outcome(
|
||||
},
|
||||
};
|
||||
assert_one_outcome(Kind::Completed, outcome, || {
|
||||
completed_usage_candidate(caches, &scope, &[], plan_complete, owner_cancelled)
|
||||
let candidate = completed_usage_candidate(caches, &scope, &[], plan_complete, owner_cancelled);
|
||||
assert_eq!(candidate.as_ref().err().copied().unwrap_or(Outcome::Accepted), outcome);
|
||||
candidate.ok()
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ An unchanged interval requires matching owner, process session, watch token, buc
|
||||
|
||||
Watch renewal keeps the original starting revision. A caller may extend its conservative local monotonic deadline after a successful renewal, but it must not replace the original proof with the renewed revision. A persisted checkpoint stores identity and revision, not a usable `Instant`; after restart, the old producer token must be checked again against the complete current owner set.
|
||||
|
||||
One keeper belongs to each leader-local cohort, with weak store and cohort references. It revisits retained watches every sixty seconds, with four concurrent queries and a five-second total limit per keepalive query. At the 256-watch limit one pass is bounded by 380 seconds, below the ten-minute producer lease. Retained checkpoints and finished sources awaiting root publication and ACK keep their original proof alive. A subsequent cycle may restart a cancelled keeper for the same instance, but cannot reset the original token, revision, or expired deadline.
|
||||
One keeper belongs to each service cohort, with weak store and cohort references. After a cycle deadline the scanner supervisor retains its cohort only when the worker has stopped, cycle state has been persisted, and the epoch has been fenced. A successful successor claim on the same storage instance can resume the cohort only from the exact fenced revision and immediately following epoch. An intervening claim, lock contention, or recovery drops the continuation. The cohort preserves its bounded watch renewals and scheduling order across the retry interval. The new leader needs its own fences for publication and dirty acknowledgements. The keeper revisits retained watches every sixty seconds, with four concurrent queries and a five-second total limit per keepalive query. At the 256-watch limit one pass is bounded by 380 seconds, below the ten-minute producer lease. Retained checkpoints and finished sources awaiting root publication and ACK keep their original proof alive. A subsequent cycle may restart a cancelled keeper for the same instance, but cannot reset the original token, revision, or expired deadline.
|
||||
|
||||
## Consumer and persistence order
|
||||
|
||||
|
||||
@@ -137,7 +137,7 @@ For batch-delete codec acceptance, run `delete_versions_request_encoding_preserv
|
||||
```bash
|
||||
cargo nextest run -p rustfs-ecstore --lib --features test-util --no-tests fail \
|
||||
-E 'test(delete_versions_request_encoding_preserves_batches_and_fleet_guards)'
|
||||
NO_PROXY='*' no_proxy='*' cargo nextest run -p rustfs --lib --features gateway-stack --no-tests fail \
|
||||
NO_PROXY='*' no_proxy='*' cargo nextest run -p rustfs --lib --no-tests fail \
|
||||
-E 'test(delete_versions_wire_roundtrip_preserves_json_and_binary_only_batches)'
|
||||
```
|
||||
|
||||
|
||||
@@ -82,7 +82,13 @@ The startup usage preflight is read-only and has a 30-second deadline. If it can
|
||||
|
||||
Cold-cache startup and startup without active replication avoid an extra bucket listing. Maintenance discovery keeps looking for replication after finding lifecycle rules in an earlier bucket. Once replication is found, both scheduling decisions are settled and remaining buckets need no startup inspection. Reapplying an unchanged resolved configuration does not reconfigure per-disk budgets or emit another configuration wakeup.
|
||||
|
||||
The `cycle_max_duration`, `cycle_max_objects`, and `cycle_max_directories` controls bound a cycle's work; they do not guarantee that every budget can complete a sweep in bounded chunks. A partial scan preserves useful checkpoints and coverage, but reopening ancestors and revalidating directory entries costs work again. A budget below the minimum replay or traversal cost can repeatedly stop at the same directory. Cross-bucket fairness also needs to be distinguished from progress within one bucket. Use pacing controls to limit background I/O, and choose cycle budgets large enough for the namespace to advance.
|
||||
The `cycle_max_duration`, `cycle_max_objects` and `cycle_max_directories` settings also apply to resumed walks. For a stable wide directory, a checkpoint records which child names have been discovered, which children have finished and the usage accumulated from those children. The next executor can restore that work after a cycle deadline or process restart. Objects from a child that was interrupted before completion are accounted for when that child finishes.
|
||||
|
||||
On Linux and macOS, the scanner can reopen a partially enumerated physical directory and verify its last saved position against the directory generation and actual entries. Each physical replica has its own position. A replacement or changed directory requires enumeration again. When an executor moves to another disk, it advances that disk's own checkpoint. Returning to the first disk resumes its saved subtotal. This keeps sizes and version counts tied to the replica whose object metadata was read. Other platforms can reuse a live directory handle and already completed work on an identifiable disk. An interrupted native directory read may need to start again after a process restart.
|
||||
|
||||
After an upgrade, an older partial traversal without a recorded physical disk starts a fresh walk. This preserves the existing published usage snapshot when the partial checkpoint is discarded. To verify a handoff, enable debug logging for `rustfs_scanner::scanner_folder::directory_cursor` and look for `state="directory_checkpoint_restored"`. The event identifies the bucket, directory, physical disk, leader epoch and cycle. `restore_mode="native_position"` records how many entries had already been read. `restore_mode="sealed_membership"` records reuse of a fully enumerated directory and includes `completed_through` when the checkpoint contains a compacted subtotal.
|
||||
|
||||
The bucket usage API can return a carried value or a newer observation while a walk is incomplete. Check `usage_value_state` and `usage_certification` to identify the last certified value. If writes overlap a resumed walk, reaching the end can produce an observation that still needs a fresh sweep before certification. Progress requires enough budget for individual I/O operations and a successful checkpoint save.
|
||||
|
||||
Budget cancellation alone does not publish complete usage or acknowledge dirty buckets. A partial cycle must retain that uncertainty; a growing directories-scanned counter is not proof of new coverage or published totals. Inspect checkpoint/frontier changes and publication status across attempts before changing budgets. A budget cancellation does not persist a disabled scheduler flag and does not justify deleting scanner state.
|
||||
|
||||
@@ -230,7 +236,7 @@ Only `xl.meta` files enter object metadata accounting. Other payload files consu
|
||||
|
||||
UUID candidates under a directory without `xl.meta` are classified during their normal directory traversal. Each directory open and entry is charged once; traversal yields every 64 entries and continues under the budget until a regular `part.N` is found or EOF is reached. A full batch is not evidence of absence. Candidate metadata accounting is deferred until classification finishes. A data directory retains the existing missing-parent-metadata and non-destructive heal path; a namespace continues from the same listing, without a second probe/open/entry pass. Data directories do not consume namespace-directory quota or a preset folder delay; a candidate proven to be a namespace consumes that quota and delay once.
|
||||
|
||||
Candidate entries use the normal raw-enumeration progress and checkpoint path. Cancellation or an iteration error leaves classification incomplete, never an EOF or successful coverage result. A child already absent or replaced by a non-directory at open is handled as vanished, without stale namespace totals or a missing-metadata heal request. A replacement retains the existing coverage gap; mid-iteration errors never become EOF. A clean namespace covered by a restored frontier reuses its prior totals only after current classification reaches EOF; metadata is not read or charged to object quota again. A later data candidate prevents completed namespace siblings from certifying the unresolved parent. Existing raw-page indexes are membership receipts, not portable filesystem seek offsets: a later cycle or process restart reopens and revalidates the directory, with actual reads charged again. This does not guarantee that a directory larger than the configured cycle budget can finish within that budget.
|
||||
Candidate entries use the normal raw-enumeration progress and checkpoint path. Cancellation or an iteration error leaves classification incomplete, never an EOF or successful coverage result. A child already absent or replaced by a non-directory at open is handled as vanished, without stale namespace totals or a missing-metadata heal request. A replacement retains the existing coverage gap; mid-iteration errors never become EOF. A clean namespace covered by a restored frontier reuses its prior totals only after current classification reaches EOF; metadata is not read or charged to object quota again. A later data candidate prevents completed namespace siblings from certifying the unresolved parent. The disk worker parks the physical enumeration between turns and checks directory identity and modification state before reuse. A cold process reopens the directory and pays for its physical reads again. Raw-page indexes identify visited names; the scanner still requires EOF, complete child accounting and a valid publication fence before certifying totals.
|
||||
|
||||
- Media is detected once per disk path from the backing block device's `rotational` flag on Linux, through partitions, `dm-*`, and software RAID; any spindle in the stack counts as rotational. Undetectable media, including non-Linux hosts and overlay filesystems without a bind-mounted volume, uses the rotational defaults because a spindle is the case that needs protection.
|
||||
- `auto` resolves to `100` operations/s and `16 MiB/s` on rotational or unknown media and to `4000` operations/s and `256 MiB/s` on flash. The rotational values leave roughly half of a 7200 rpm spindle's random reads to foreground traffic; the flash values are a few percent of a SATA SSD. The `fastest` speed preset resolves `auto` to no cap; explicit numbers always apply.
|
||||
@@ -313,6 +319,7 @@ metrics.maintenance_control.primary_control
|
||||
metrics.source_work
|
||||
metrics.replication_repair
|
||||
metrics.scan_checkpoint
|
||||
metrics.last_cycle_partial_stage
|
||||
metrics.cycle_timeout_total
|
||||
metrics.cycle_last_progress_age
|
||||
metrics.leader_lease_without_progress
|
||||
@@ -335,6 +342,8 @@ catch_up_estimate.discovered_expiry_items
|
||||
catch_up_estimate.discovered_transition_items
|
||||
```
|
||||
|
||||
For a partial cycle, `metrics.last_cycle_partial_stage` records the blocking check for that cycle. For example, `entry_budget` counts raw directory enumeration, `source_result_missing` means an expected set supplied no result, and `activity_baseline_unverified` means the activity proof could not certify publication. A `mixed_plan_recertification_pending` stage means every outstanding bucket completed traversal and supplied a fresh cross-plan observation that still requires certification. Candidate rejections retain the existing candidate outcome labels such as `invalid_root` and `tier_proof`. The field clears on the next non-partial cycle outcome. Inspect `metrics.usage_publication` for the outcome of writing the root snapshot. When a cycle ends with incomplete coverage or an unverifiable observation, the terminal error event includes the cycle, leader epoch and blocking stage. Budget stops and saved checkpoints remain visible in scanner status and warning logs. A supervisor deadline that interrupts the worker before a terminal result increments `metrics.cycle_timeout_total`.
|
||||
|
||||
### Deep scan cadence state
|
||||
|
||||
`bitrot` reports how the scanner decided the scan mode of the running or most recent cycle:
|
||||
@@ -361,6 +370,8 @@ A resumed walk retains its previous durable coverage while initial enumeration h
|
||||
|
||||
Shared compacted accumulators defer periodic emission until the owning subtree completes; replay inside that compacted subtree remains possible after an interruption. When bucket checkpoint preparation returns `Reset` for existing resume state, it logs `checkpoint_reset` with the validation reason. The `scan_checkpoint_used` counter includes validated coverage adoption by the forward sweep.
|
||||
|
||||
Fenced per-bucket checkpoints keep small payloads inline. Large checkpoints stage immutable, checksum-addressed pages and then conditionally replace a small manifest. The primary and backup use separate page objects. Interruption before the manifest write leaves the prior checkpoint readable. Cold recovery verifies the referenced pages and can carry verified restore progress across local time slices. A reader that predates the page format can rebuild a partial checkpoint and can read compact complete totals when the compatibility projection fits the manifest.
|
||||
|
||||
A confirmed periodic save does not prove restart resumability. Adoption still validates bucket/set identity and, for non-empty caches, connected cache scope. Skipping completed subtrees additionally requires a validated coverage frontier; raw enumeration state alone does not prove completed subtree coverage.
|
||||
|
||||
### Cycle recovery on followers
|
||||
@@ -898,42 +909,6 @@ done
|
||||
|
||||
The `aggregated.scanner` payload preserves the same scanner progress, checkpoint, pacing, source work, maintenance control, lifecycle expiry, and lifecycle transition fields used by the local scanner status, but only for the responding node; `by_host.*.scanner` keeps that node's host view. Compare the per-node artifacts externally to find old active paths, partial checkpoints, pacing pressure, source-level control pressure, or downstream queue admission problems across the deployment.
|
||||
|
||||
## Reading Cached Bucket Statistics
|
||||
|
||||
Admin usage reads cache the selected scanner baseline for the existing 30-second TTL, then verify every returned bucket against uncached storage-owned incarnation metadata. Identity reads use at most four concurrent operations and a shared two-second deadline after releasing snapshot/refresh locks. Each bucket read linearizes independently against metadata transactions; the response is not an atomic snapshot of the entire namespace. Delayed peer invalidation cannot authorize a predecessor's statistics for a recreated bucket.
|
||||
|
||||
Missing, failed, timed-out, nil, mismatched, or legacy absent identities leave the bucket value absent (unknown), rather than returning the predecessor's value or a confirmed zero. Legacy statistics become visible after a scanner snapshot supplies the current identity. Account-info and usage display apply the existing single-process memory overlay before the final identity filter; distributed local counters are not promoted. When a projection omits values or changes an overlay, totals are partial and `usage_snapshot_converged=false`, and per-tier/replication aggregates are unavailable. Verified cold bucket values remain visible. Such response-only projections cannot be persisted, used for quota admission, or acknowledge dirty work.
|
||||
|
||||
## Reading Scanner Work Observations
|
||||
|
||||
`GET /rustfs/admin/v3/scanner/status` exposes `metrics.scan_observation` for the responding node. Capture timestamped snapshots on every node, including the scanner leader, and compare their deltas over the same workload window. These process-lifetime counters reset on restart; a missing field on an older node means unobserved. They are loaded independently and may briefly disagree while operations finish. This telemetry does not change scan cadence, cached usage reads, quota authority, dirty acknowledgement, or maintenance requirements.
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `metadata_reads.attempts` | Disk metadata read futures started after scanner node admission. Invalid paths and operations cancelled before starting the read do not count. |
|
||||
| `metadata_reads.succeeded`, `not_found`, `errors`, `cancelled` | Storage outcomes, including object/version disappearance and dropped read futures. Successful storage reads can still contain corrupt metadata; decoding and lifecycle failures do not become read errors. Cancellation observes the future, not proof that physical I/O stopped. |
|
||||
| `metadata_reads.bytes` | Total returned buffer lengths for successful reads, including inline data and corrupt metadata. Failed reads contribute no known bytes; this is not physical disk traffic. |
|
||||
| `metadata_reads.elapsed_nanos` | Sum of read-future elapsed time, including disk-permit or RPC waits, failures, and cancellation. Folder pacing, scanner node admission, decoding, and lifecycle work are outside this interval. Concurrent reads overlap, so the sum is not cycle wall time. |
|
||||
| `cache_reuse.clean_prefix` | Clean root subtrees copied under an admitted prefix scope by the non-erasure scanner. |
|
||||
| `cache_reuse.covered_namespace` | Namespace subtrees copied from validated checkpoint coverage. |
|
||||
| `cache_reuse.sampled_namespace` | Cached namespace subtrees copied when the existing periodic sampling selected other work. |
|
||||
| `cache_reuse.classified_namespace` | EC candidate namespace subtrees copied only after the current listing finished classifying them. Health/classification work still runs. |
|
||||
| `cache_reuse.*.count`, `elapsed_nanos` | Completed subtree reuse decisions and time spent copying/attaching their cache entries. Counts are not objects skipped or a cache hit ratio; times exclude classification/listing and are not total cache maintenance cost. |
|
||||
| `bucket_reuse.scoped_cold_bucket` | Whole cold buckets copied and attached to a successfully prepared scoped set cache. Count is copied buckets, not plans selected; later turn publication can still fail. Time covers copying/attachment only. |
|
||||
| `bucket_reuse.current_root` | Existing current bucket roots successfully sent to the local collector after watch, namespace lease and publication epoch checks. Time covers channel send only; failure or cancellation contributes no completed reuse. Collector acceptance and durable publication remain separate. |
|
||||
| `reuse_decisions.*.not_evaluated`, `blocked`, `eligible` | First conditions reached by production clean-prefix, checkpoint-coverage and namespace-sampling gates. Policy exclusions (`missing_scope`, `no_resume_sweep`, `deep_scan`, `forward_sweep`, `active_replication`) precede cache proof checks. `blocked` reports the first failed work/cache condition; later checks are not repeated for diagnostics. `eligible` does not certify a copy: an EC candidate can still require classification. |
|
||||
| `erasure_clean_prefix_walks` | Clean root children that still require walking because EC scans perform independent object-health work. |
|
||||
| `prefix_scope_resolution.evaluated`, `activated`, `blockers` | Proof evaluations retained by the final scope resolver, admitted proof gates, and every failed prerequisite. An admitted proof does not guarantee a reusable child exists; actual subtree copies appear in `cache_reuse`. Startup/required full scans, early scope rejection, and discarded peer/ACK resolutions do not count here. |
|
||||
| `prefix_cycle_completion.evaluated`, `activated`, `blockers` | End-of-cycle proof evaluations for subsequent reuse. An activated completion proof does not mean this cycle reused prefixes or that its later publication/persistence succeeded. One evaluation can increment several blocker counters. |
|
||||
| `scope_fallbacks` | Per-reason breakdown of the existing `usage_freshness.scope_fallback_total`; required full scans are excluded. |
|
||||
| `authoritative_baseline_rejections`, `observed_baseline_rejections` | First failed validation checks for `baseline_unavailable` fallback cycles, preserving the authoritative rejection when the observed fallback also fails. Missing/undecodable authority never reaches observed validation; `authoritative_identity` means it could not anchor an observed fallback. |
|
||||
|
||||
`freshness` describes the responding node's completed-cycle age, not whether every bucket has incorporated new writes. Read `metrics.usage_freshness` for node-local live pending work, guardrail vetoes, durable success and publication state; read `metrics.usage_publication` for the confirmed snapshot's pending count and publication class/outcome, and `cycle_schedule` for effective retry intervals and consecutive superseded cycles. Missing or unknown publication data does not establish zero pending work. A follower or a single node's empty dirty map cannot establish cluster-wide convergence.
|
||||
|
||||
Whole cold bucket copies, current-root reuse and the four subtree categories have different prerequisites and are not interchangeable hit rates. EC clean prefixes retain health walks. Deep or required full work must run, but a completed cache matching that exact work digest may satisfy a retry of the same cycle. A normal workload need not exercise current-root reuse or every subtree category; use controlled tests for positive emission evidence and capture real gate reasons when a VM window has zero copies.
|
||||
|
||||
The existing `life_time_ops.read_metadata` and `last_minute.actions.read_metadata` now include finished and cancelled read attempts; the latter reports count, accumulated whole seconds, and returned bytes in its existing window. Use `metadata_reads.elapsed_nanos` for subsecond read costs. Use read-count and byte deltas to assess work growth, and elapsed-time deltas to assess read-future cost. Compare those with reused-subtree deltas, scope fallback reasons, cycle wall time, pacing waits, and source work before attributing freshness delays to metadata I/O or proposing a scan optimization.
|
||||
|
||||
## Reading Lifecycle Transition Status
|
||||
|
||||
`metrics.lifecycle_transition`:
|
||||
@@ -968,7 +943,7 @@ For a mostly idle single-node, single-disk deployment with sustained CPU usage w
|
||||
6. If `primary_pressure` is `throttle_pause` and pause ratios are low, raise `scanner.delay` first.
|
||||
7. If individual sleeps are too short, raise `scanner.max_wait`.
|
||||
8. If each scan cycle finishes but starts too often, raise `scanner.cycle`.
|
||||
9. To bound one cycle, set `scanner.cycle_max_duration`, `scanner.cycle_max_objects`, or `scanner.cycle_max_directories`; confirm that checkpoints advance under that budget. These controls do not guarantee resumable bounded chunks for every directory shape; see [Startup and cycle budgets](#startup-and-cycle-budgets).
|
||||
9. To bound one cycle, set `scanner.cycle_max_duration`, `scanner.cycle_max_objects`, or `scanner.cycle_max_directories`; confirm that checkpoints advance under that budget. For the conditions that permit a wide directory to advance across turns and process restarts, see [Startup and cycle budgets](#startup-and-cycle-budgets).
|
||||
10. Recheck `pacing_pressure`, `maintenance_control`, source work, and lifecycle transition status after one or more scanner cycles.
|
||||
|
||||
Do not rely only on a longer cycle interval if lifecycle, replication, heal, or bitrot work must keep moving; use source work and transition status to confirm that background maintenance still progresses.
|
||||
|
||||
@@ -56,13 +56,11 @@ The four site-replication state-writer concurrency proofs reserve the available
|
||||
| PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
|
||||
| Full selection | `Workspace tests` (serial ILM step) | `ci.yml` `workspace-tests` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job |
|
||||
| Nightly schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo nextest run` with the job's `--features` |
|
||||
| Full selection, every event | `Test and Lint (gateway-stack)` | `ci.yml` `test-and-lint-gateway-stack` | Via aggregate | `cargo clippy -p rustfs --all-targets --features gateway-stack -- -D warnings`; `cargo nextest run -p rustfs --features gateway-stack` with the job's filter |
|
||||
| Nightly schedule, manual dispatch, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `scripts/check_connect_release_boundary.py`: `cargo check -p rustfs --release` with `connect-e2e-short-credentials` and with `gateway-stack` must each fail at its guard; `scripts/check_offline_enrollment_e2e.sh` |
|
||||
| Nightly schedule, manual dispatch, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `scripts/check_connect_release_boundary.py`: `cargo check -p rustfs --release` with `connect-e2e-short-credentials` must fail at its guard; `scripts/check_offline_enrollment_e2e.sh` |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) |
|
||||
| Nightly schedule, manual dispatch, full selection | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
| PR, non-doc change | `End-to-End Tests` | `ci.yml` `e2e-tests` | Via aggregate | `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test`, then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` under the same wrapper; membership guards `scripts/check_test_wiring.py --check-profile e2e-smoke <listing.json>` and `scripts/check_security_smoke_count.sh check <listing.json>` |
|
||||
| PR, non-doc change | `Core contract` | `ci.yml` `core-contract` | Via aggregate | existing write/read, anonymous denial and restart-byte tests via `core-contract` profile and the E2E archive |
|
||||
| Push to `main` or `gateway/integration`, merge group, nightly schedule, manual dispatch; PR only when it changes a gateway-stack input (`GATEWAY_SMOKE_PATHS` or a `rustfs-gateway*` pin, `scripts/ci_gate.py`) | `e2e-smoke (gateway, report-only until T2.10)` | `ci.yml` `e2e-tests-gateway` | Report-only until rustfs/backlog#2734 T2.10 makes it required; `continue-on-error`, outside the aggregate; skipped, never green, on PRs that touch no gateway-stack input. At T2.10, when the lane becomes required, it must run on every pull request again — delete this selection, do not extend it | `RUSTFS_S3_STACK=gateway python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test`; `scripts/ci_gate.py` keeps its archive and run steps identical to `e2e-tests`, which sets `RUSTFS_S3_STACK=legacy` on this branch |
|
||||
| PR, non-doc change | `S3 Implemented Tests` | `ci.yml` `s3-implemented-tests` | Via aggregate | build `rustfs`, then `scripts/s3-tests/run.sh` with the job's `DEPLOY_MODE` / `TEST_MODE` / `MAXFAIL` env |
|
||||
| PR, non-doc change | `S3 Lifecycle Behavior Tests` | `ci.yml` `s3-lifecycle-behavior-tests` | Via aggregate | `scripts/s3-tests/run.sh` with the job's accelerated-scanner env |
|
||||
| PR touching `paths` in `audit.yml` | `Cargo Deny`, `Workflow Pin Report`, `Dependency Review` | `audit.yml` `cargo-deny`, `workflow-pin-report`, `dependency-review` | Report-only | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
@@ -93,7 +91,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
|
||||
| `cache-warm.yml` (every six hours, plus main pushes that change `Cargo.lock`, `rust-toolchain.toml` or the setup action) | `warm-ci-dev`, `warm-ci-feat-rio`, `warm-ci-feat-proto`, `warm-ci-uring` | no verdict; sole writer of the `rust-cache` keys that `ci.yml` restores | yes | dispatch `cache-warm.yml` |
|
||||
| `build.yml` (nightly Linux, weekly desktop) | `build-rustfs` over the `prepare-platform-matrix` subset for the firing cron: four Linux targets at 01:13 UTC, and macOS aarch64 plus Windows x86_64 at Sunday 16:17 UTC | build/package integrity | yes | dispatch `build.yml` with an exact platform set |
|
||||
| `e2e-replication-nightly.yml` (nightly) | `repl-nightly`, `cluster-nightly`, `protocols-nightly` | three independent gates; JUnit, membership listing, server logs | yes | `python3 scripts/e2e_binary.py build --bins`, then `python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--features e2e-test-hooks` on both steps with `--profile e2e-nightly`; `--features ftps,webdav,sftp` on both steps with `-j 1 --profile e2e-protocols` |
|
||||
| `dual-stack-nightly.yml` (nightly) | `build`, `leg` (`e2e-smoke` and `e2e-full`, each on the legacy and, once a gateway start is proven, the gateway stack), `summary` | report-only both-stack diff of the two profiles only `ci.yml` runs; the other e2e profiles carry the same `stack` dimension in their own workflows. A gateway binary that refuses to start skips the gateway legs and the summary says why; the gateway legs are report-only until T2.10 | yes | `python3 scripts/dual_stack.py summary --help`; dispatch `dual-stack-nightly.yml` |
|
||||
| `dual-stack-nightly.yml` (nightly) | `build`, `leg` (`e2e-smoke` and `e2e-full`) | the two profiles only `ci.yml` runs, nightly on the gateway stack (the workflow keeps its name from the both-stack period); per-profile JUnit and server logs retained 30 days. The one comparison against the last legacy record is made by hand with `scripts/junit_diff.py` | yes | dispatch `dual-stack-nightly.yml` |
|
||||
| `e2e-distributed.yml` (nightly) | `distributed` | fail-closed 4-node 4-disk S3, durability, replication, movement, fault, and direct/rolling upgrade gate; JUnit, membership listing, per-node server logs | yes | download the pinned previous release as in the workflow, export `RUSTFS_UPGRADE_SOURCE_BINARY`, run `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks`, then `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-distributed -p e2e_test` |
|
||||
| `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target |
|
||||
| `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes (disabled) | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
|
||||
@@ -101,12 +99,12 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run. The v
|
||||
| `on-demand-migration-interop.yml` (nightly) | `minio-source`, `cloud-source` (`aws`, `r2`, `gcs`) | report-only provider interop; one JSON report per provider naming cases, timings and source request counts, plus JUnit and MinIO logs. A cloud provider whose `ODM_INTEROP_*` secrets are absent is skipped with a summary note, not failed | yes | start the pinned MinIO container as in the job, export the `RUSTFS_ODM_INTEROP_*` variables, run `python3 scripts/e2e_binary.py build --bins`, then `python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test` |
|
||||
| `nightly-gnu.yml` (nightly) | `build`, `kms-vault-lane`, `kms-vault-ha-failover` | build, live Vault, and HA failover gates | yes | commands and pinned Vault images in the workflow |
|
||||
| `audit.yml` (nightly) | `cargo-deny`, `workflow-pin-report` | dependency and workflow-pin gates | yes | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
|
||||
| `mint.yml` (weekly) | `image`, `mint` (legacy and gateway), `compare` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json` per stack. The image is built once. `compare` holds the gate that every case legacy passes is present and passing on gateway and prints the cases that break it; the gateway leg and `compare` are report-only until T2.10 | yes | pinned Docker sequence in the workflow; `python3 scripts/mint_stack_diff.py <legacy-leg-dir> <gateway-leg-dir>` |
|
||||
| `mint.yml` (weekly) | `image`, `mint` | report-only by design; per-suite PASS/FAIL/NA and raw `log.json`. The image is built once | yes | pinned Docker sequence in the workflow |
|
||||
| `coverage.yml` (weekly) | `coverage` | report-only trend; lcov and JSON artifact | yes (disabled) | `make coverage` |
|
||||
| `runner-hygiene.yml` (monthly) | `check-ephemerality` | runner ephemerality | yes | dispatch |
|
||||
| `e2e-upgrade.yml` (weekly, and on version tags) | `upgrade` (serial cases sharing one harness build) | upgrade and rollback gate; server logs | yes | `python3 scripts/e2e_binary.py build`, then the job's `python3 scripts/e2e_binary.py run -- cargo test --locked -p e2e_test` commands with `RUSTFS_UPGRADE_SOURCE_BINARY` pointing at the pinned previous release (`UPGRADE_SOURCE_VERSION`) |
|
||||
| `reference-parity.yml` (nightly, and on PRs that change the lane's own files) | `plan`, `build-candidate`, `s3tests` (one job per target in `.config/reference-parity-targets.toml`), `compare` | report-only parity gate: fails on any per-node-id s3-tests difference between a branch candidate and the release pinned in `.config/reference-binary.toml` that `.config/reference-parity-tolerated.txt` does not list, and on a stale tolerated entry; per-target JUnit, logs and `diff.md`/`diff.json` | yes | `DEPLOY_MODE=binary RUSTFS_BINARY=<binary> MAXFAIL=0 XDIST=4 ./scripts/s3-tests/run.sh` once per target, then `python3 scripts/s3-tests/reference_parity.py compare` as in the `compare` job |
|
||||
| `gateway-acceptance.yml` (nightly) | `plan`, `build-candidate`, `build-gateway-tools`, `acceptance` (one job per S3 stack), `compare` | the rustfs/gateway conformance corpus (`--profile rustfs`) and client matrix (`--external`), both checked out at the rev `Cargo.toml` pins. Each stack must be observed in the server's own `server_starting` log event (`scripts/dual_stack.py stack`). Legacy stack: the corpus run is a reference that is never judged (it must serve, stay up and complete); the matrix is the baseline and fails on a `fail` or `sut-unregistered` cell or a client that could not be installed. Gateway stack, report-only until T2.10: fails on a conformance failure without a `.config/conformance-rulings.toml` row, on a ruled case failing differently, on a stale row, on a skip on the server's own answer, and on any matrix cell that passes on legacy but not on gateway. Per-stack `report.json`, `matrix.json` and logs, and a legacy-vs-gateway diff | yes | start the candidate with `RUSTFS_S3_STACK=<stack>`, then the job's `rustfs-gateway-conformance run` and `ci/compat/run_matrix.sh --external` commands, then `python3 scripts/check_conformance_rulings.py` and `python3 scripts/compat_matrix_report.py check` |
|
||||
| `gateway-acceptance.yml` (nightly) | `plan`, `build-candidate`, `build-gateway-tools`, `acceptance` | the rustfs/gateway conformance corpus (`--profile rustfs`) and client matrix (`--external`), both checked out at the rev `Cargo.toml` pins, against one candidate server. Fails on a conformance failure without a `.config/conformance-rulings.toml` row, on a ruled case failing differently, on a stale row, on a skip on the server's own answer, and on a matrix `fail` or `sut-unregistered` cell or a client that could not be installed. `report.json`, `matrix.json` and logs | yes | start the candidate, then the job's `rustfs-gateway-conformance run` and `ci/compat/run_matrix.sh --external` commands, then `python3 scripts/check_conformance_rulings.py --run-dir <dir>` and `python3 scripts/compat_matrix_report.py check --run-dir <dir>` |
|
||||
| `oidc-keycloak.yml` (weekly) | `oidc-keycloak-live` | live OIDC gate | yes | see the PR row |
|
||||
| `targets-integration.yml` (nightly) | `targets-live` | live target gate; container logs | yes (disabled) | see the PR row |
|
||||
| `gateway-integration-sync.yml` (weekdays) | `sync` | merges `main` into `gateway/integration` as a `--no-ff` merge commit; a conflicting merge is aborted and opens a `sync/<source>-<date>` pull request listing the conflicting files, which keeps the run green; a rejected push or missing pull request fails it | yes | dispatch, optionally with `source_ref` |
|
||||
|
||||
@@ -64,7 +64,7 @@ A coverage receipt binds the completed traversal frontier to its scope, starting
|
||||
|
||||
The Normal-to-Deep regression holds the mutation plan fixed, changes metadata in a previously visited prefix, and verifies that the real Deep disk-scan entry point reads that prefix again. It also checks that a complete Normal cache cannot satisfy the Deep `Current` path. The fixture disables heal side effects; it proves traversal re-entry, not actual bitrot detection or repair. Additional tests cover map round-trips, stale identities, existing-but-uncovered cursors, coverage gaps, and per-instance metadata ownership.
|
||||
|
||||
This fixture bounds object processing after directory enumeration. It does not prove fixed-budget enumeration of arbitrarily wide directories or real process-restart convergence. Those gates require a storage-owned resumable enumeration capability, including its initial construction cost; a readdir offset, an in-memory iterator or a last-name filter is not that capability.
|
||||
The independent `wide_directory_runtime_budget_reaches_eof` regression charges virtual time for every physical directory read, including replay and EOF, and checks exact object and byte totals across capped rounds. The `paged_checkpoint` tests exercise immutable page reuse, interrupted root commits, stale writers and cold restoration. The process-level regression in [the E2E crate](../../crates/e2e_test/src/scanner_wide_directory_test.rs) seeds objects through S3, restarts the server after a partial save, and checks the certified admin usage against a paginated S3 listing using the same scanner budget on both boots.
|
||||
|
||||
For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store.
|
||||
|
||||
|
||||
+6
-15
@@ -56,12 +56,10 @@ required-features = ["swift"]
|
||||
name = "swift_object_integration_test"
|
||||
required-features = ["swift"]
|
||||
|
||||
# Starts an embedded server on the gateway stack, which only a build with the
|
||||
# feature can select.
|
||||
# Starts an embedded server on the gateway stack.
|
||||
[[test]]
|
||||
name = "embedded_gateway_stack_test"
|
||||
path = "tests/embedded_gateway_stack_test/main.rs"
|
||||
required-features = ["gateway-stack"]
|
||||
|
||||
[features]
|
||||
internode-transport = ["rustfs-ecstore/internode-transport", "rustfs-rio/internode-transport"]
|
||||
@@ -77,19 +75,12 @@ license = []
|
||||
io-scheduler-debug = [] # Enable debug information in I/O scheduler
|
||||
tracing-chunk-debug = [] # Enable per-chunk tracing in data plane (high noise, for debugging only)
|
||||
full = ["metrics-gpu", "ftps", "swift", "webdav", "sftp", "pyroscope", "gcs"]
|
||||
# The debug-only E2E binary also carries the gateway S3 stack, so the gateway smoke
|
||||
# lane runs the same artifact as the legacy one with RUSTFS_S3_STACK as its only change.
|
||||
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks", "gateway-stack"]
|
||||
e2e-test-hooks = ["rustfs-ecstore/e2e-test-hooks", "rustfs-scanner/e2e-test-hooks"]
|
||||
# Shortens Connect credentials only in debug E2E builds.
|
||||
connect-e2e-short-credentials = []
|
||||
# Builds the dedicated rustfs-cli-e2e target with a build-time public enrollment root.
|
||||
offline-enrollment-e2e-root = []
|
||||
rio-v2 = ["rustfs-ecstore/rio-v2"]
|
||||
# The rustfs-gateway S3 stack, selected at startup with RUSTFS_S3_STACK=gateway
|
||||
# (rustfs/backlog#2734). Under construction: off by default and never enabled in a
|
||||
# release build; the stack refuses to compile without debug assertions, and
|
||||
# `scripts/check_connect_release_boundary.py` keeps every release-reachable feature from enabling it.
|
||||
gateway-stack = ["dep:rustfs-gateway", "dep:rustfs-gateway-adapter", "dep:rustfs-gateway-core", "dep:rustfs-gateway-dialect-rustfs-admin"]
|
||||
# Linux-only cuObject S3-over-RDMA GET into client (GPU) memory; needs libcuobjserver.
|
||||
cuobject = ["dep:rustfs-cuobject", "rustfs-cuobject/server"]
|
||||
# NIXL/UCX transport for S3-over-RDMA; replaces cuobject when both are enabled.
|
||||
@@ -279,10 +270,10 @@ rustfs-notify = { workspace = true }
|
||||
rustfs-obs = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
rustfs-protocols = { workspace = true }
|
||||
rustfs-gateway = { workspace = true, optional = true }
|
||||
rustfs-gateway-adapter = { workspace = true, optional = true }
|
||||
rustfs-gateway-core = { workspace = true, optional = true }
|
||||
rustfs-gateway-dialect-rustfs-admin = { workspace = true, optional = true }
|
||||
rustfs-gateway = { workspace = true }
|
||||
rustfs-gateway-adapter = { workspace = true }
|
||||
rustfs-gateway-core = { workspace = true }
|
||||
rustfs-gateway-dialect-rustfs-admin = { workspace = true }
|
||||
rustfs-gateway-stream = { workspace = true }
|
||||
rustfs-gateway-types = { workspace = true, features = ["compat-s3s-rustfs"] }
|
||||
rustfs-protos = { workspace = true }
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::admin::service::caller_identity::oidc_profile_fields;
|
||||
use crate::admin::storage_api::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::admin::storage_api::contract::admin::StorageAdminApi;
|
||||
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
|
||||
use crate::admin::storage_api::data_usage::load_admin_data_usage_with_memory_overlay;
|
||||
use crate::admin::storage_api::data_usage::{apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached};
|
||||
use crate::admin::storage_api::metadata_sys;
|
||||
use crate::auth::get_condition_values;
|
||||
use crate::server::ADMIN_PREFIX;
|
||||
@@ -278,7 +278,8 @@ impl Operation for AccountInfoHandler {
|
||||
// process-local absolute counters to cluster-wide bucket totals.
|
||||
// This path never triggers a live full-version listing
|
||||
// (rustfs/backlog#1306); freshness is owned by the scanner.
|
||||
let data_usage_info = map_data_usage_result(load_admin_data_usage_with_memory_overlay(store.clone()).await)?;
|
||||
let mut data_usage_info = map_data_usage_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
|
||||
apply_bucket_usage_memory_overlay(&mut data_usage_info).await;
|
||||
account_info.usage_snapshot_converged = data_usage_info.usage_snapshot_converged;
|
||||
|
||||
for bucket in buckets.iter() {
|
||||
|
||||
@@ -1261,7 +1261,16 @@ async fn table_storage_credentials_are_rejected_by_catalog_authentication() {
|
||||
}
|
||||
let mut credential = get_new_credentials_with_metadata(&claims, root_secret).expect("sign a temporary credential");
|
||||
credential.parent_user = parent.to_string();
|
||||
iam.set_temp_user(&credential.access_key, &credential, None)
|
||||
// A regular parent's sessions must carry its durable STS generation, so publish
|
||||
// through the parent-bound path instead of storing an unversioned session.
|
||||
let parent_secret = if parent == root_key { root_secret } else { parent_secret };
|
||||
let authenticated_parent = rustfs_credentials::Credentials {
|
||||
access_key: parent.to_string(),
|
||||
secret_key: parent_secret.to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
iam.set_temp_user_from_parent(&credential, &authenticated_parent)
|
||||
.await
|
||||
.expect("store temporary credential");
|
||||
let request = load_table_request_builder(context.clone(), &credential.access_key, &credential.secret_key)
|
||||
|
||||
@@ -916,14 +916,12 @@ async fn handle_object_lambda_request(req: &mut S3Request<Body>, bucket: &str, o
|
||||
invoke_object_lambda_target(req, bucket, object, get_resp).await
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
/// The bucket and key of an Object Lambda request, which takes precedence over every other route
|
||||
/// this router claims; `None` for any other request.
|
||||
pub(crate) fn object_lambda_target(method: &Method, uri: &Uri) -> Option<(String, String)> {
|
||||
parse_object_lambda_request(method, uri)
|
||||
}
|
||||
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
/// Serves an admitted Object Lambda request that another edge verified, through the same handler
|
||||
/// the legacy `call` runs.
|
||||
pub(crate) async fn serve_object_lambda(
|
||||
@@ -973,7 +971,6 @@ pub(crate) fn console_listener_route(method: &Method, uri: &Uri, headers: &Heade
|
||||
|
||||
/// Whether this router answers the request itself rather than an S3 operation: the RustFS extension
|
||||
/// routes and every control-plane route.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) fn claims_route(method: &Method, uri: &Uri, headers: &HeaderMap) -> bool {
|
||||
s3_extension::route_key(method, uri).is_some()
|
||||
|| parse_object_lambda_request(method, uri).is_some()
|
||||
@@ -981,7 +978,6 @@ pub(crate) fn claims_route(method: &Method, uri: &Uri, headers: &HeaderMap) -> b
|
||||
}
|
||||
|
||||
/// Signed system-status and catalog-configuration reads, after object-lambda routing.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) fn claims_native_admin_route(method: &Method, uri: &Uri) -> bool {
|
||||
method == Method::GET
|
||||
&& matches!(
|
||||
@@ -1011,7 +1007,6 @@ pub(crate) fn claims_native_admin_route(method: &Method, uri: &Uri) -> bool {
|
||||
}
|
||||
|
||||
/// Console paths whose requests do not belong to an earlier extension route.
|
||||
#[cfg(feature = "gateway-stack")]
|
||||
pub(crate) fn claims_native_console_route(method: &Method, uri: &Uri) -> bool {
|
||||
is_console_path(uri.path())
|
||||
&& parse_object_lambda_request(method, uri).is_none()
|
||||
|
||||
@@ -945,6 +945,10 @@ pub(crate) static ERR_TIER_RESERVED_NAME: AdminErrorRef = AdminErrorRef(|| &ecst
|
||||
pub(crate) mod data_usage {
|
||||
use std::sync::Arc;
|
||||
|
||||
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
|
||||
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
|
||||
}
|
||||
|
||||
pub(crate) async fn load_data_usage_from_backend(
|
||||
store: Arc<crate::storage::storage_api::ECStore>,
|
||||
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
|
||||
@@ -956,12 +960,6 @@ pub(crate) mod data_usage {
|
||||
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
|
||||
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
|
||||
}
|
||||
|
||||
pub(crate) async fn load_admin_data_usage_with_memory_overlay(
|
||||
store: Arc<crate::storage::storage_api::ECStore>,
|
||||
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
|
||||
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_with_memory_overlay(store).await
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) mod access {
|
||||
|
||||
@@ -21,7 +21,9 @@ use super::storage_api::admin_usecase::capacity::{
|
||||
};
|
||||
use super::storage_api::admin_usecase::contract::StorageAdminApi;
|
||||
use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
|
||||
use super::storage_api::admin_usecase::data_usage::load_admin_data_usage_with_memory_overlay;
|
||||
use super::storage_api::admin_usecase::data_usage::{
|
||||
apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached,
|
||||
};
|
||||
use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
|
||||
use crate::app::runtime_sources::{
|
||||
AppContext, current_app_context, current_endpoints_handle, current_object_store_handle_for_context,
|
||||
@@ -270,7 +272,7 @@ impl DefaultAdminUsecase {
|
||||
info.buckets_usage.retain(|bucket, _| current_buckets.contains(bucket));
|
||||
info.bucket_sizes.retain(|bucket, _| current_buckets.contains(bucket));
|
||||
|
||||
if info.buckets_usage.is_empty() && !current_buckets.is_empty() && !info.usage_snapshot_admin_projection {
|
||||
if info.buckets_usage.is_empty() && !current_buckets.is_empty() {
|
||||
*info = DataUsageInfo::default();
|
||||
return;
|
||||
}
|
||||
@@ -285,14 +287,19 @@ impl DefaultAdminUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep already identity-bound display values without accepting a display
|
||||
/// projection as a persistable or quota-authoritative snapshot.
|
||||
fn present_identity_bound_data_usage_snapshot(
|
||||
mut info: DataUsageInfo,
|
||||
buckets: impl IntoIterator<Item = String>,
|
||||
) -> DataUsageInfo {
|
||||
let snapshot_measured =
|
||||
info.usage_snapshot_admin_projection || info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot();
|
||||
/// Overlay single-process counts on a persisted snapshot and narrow it
|
||||
/// for display.
|
||||
///
|
||||
/// Whether the snapshot measured anything is decided before the overlay.
|
||||
/// The overlay clears `usage_snapshot_complete` when it holds counts it
|
||||
/// cannot reconcile with the snapshot, such as writes into a bucket that
|
||||
/// has no verified baseline. That makes the totals unconverged, not the
|
||||
/// scanned buckets unusable; treating it as structural incompleteness
|
||||
/// blanked usage for every bucket for as long as one such bucket kept
|
||||
/// receiving writes, because each write restarts its scanner fence.
|
||||
async fn present_data_usage_snapshot(mut info: DataUsageInfo, buckets: impl IntoIterator<Item = String>) -> DataUsageInfo {
|
||||
let snapshot_measured = info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot();
|
||||
apply_bucket_usage_memory_overlay(&mut info).await;
|
||||
Self::narrow_data_usage_snapshot_to_measured_buckets(&mut info, snapshot_measured, buckets);
|
||||
info
|
||||
}
|
||||
@@ -338,7 +345,7 @@ impl DefaultAdminUsecase {
|
||||
/// This path never triggers a live full-version listing
|
||||
/// (rustfs/backlog#1306); freshness is owned by the scanner.
|
||||
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
|
||||
let info = Self::map_data_usage_load_result(load_admin_data_usage_with_memory_overlay(store.clone()).await)?;
|
||||
let info = Self::map_data_usage_load_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
|
||||
let buckets = store
|
||||
.list_bucket(&BucketOptions {
|
||||
cached: true,
|
||||
@@ -347,7 +354,7 @@ impl DefaultAdminUsecase {
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "list_bucket failed"))?;
|
||||
let mut info = Self::present_identity_bound_data_usage_snapshot(info, buckets.into_iter().map(|bucket| bucket.name));
|
||||
let mut info = Self::present_data_usage_snapshot(info, buckets.into_iter().map(|bucket| bucket.name)).await;
|
||||
|
||||
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
|
||||
|
||||
@@ -972,52 +979,6 @@ mod tests {
|
||||
assert_eq!(info.buckets_count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_bound_partial_display_keeps_valid_buckets_and_unknown_state() {
|
||||
use std::collections::HashMap;
|
||||
|
||||
let mut projected = DataUsageInfo {
|
||||
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
|
||||
usage_snapshot_admin_projection: true,
|
||||
usage_snapshot_partial: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
buckets_count: 1,
|
||||
objects_total_count: 7,
|
||||
objects_total_size: 70,
|
||||
buckets_usage: HashMap::from([(
|
||||
"cold".to_string(),
|
||||
rustfs_data_usage::BucketUsageInfo {
|
||||
size: 70,
|
||||
objects_count: 7,
|
||||
..Default::default()
|
||||
},
|
||||
)]),
|
||||
bucket_sizes: HashMap::from([("cold".to_string(), 70)]),
|
||||
..Default::default()
|
||||
};
|
||||
projected = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(
|
||||
projected,
|
||||
["cold".to_string(), "recreated".to_string()],
|
||||
);
|
||||
assert_eq!(projected.buckets_usage["cold"].objects_count, 7);
|
||||
assert!(!projected.buckets_usage.contains_key("recreated"));
|
||||
assert_eq!(projected.objects_total_count, 7);
|
||||
assert_eq!(projected.usage_snapshot_converged, Some(false));
|
||||
assert!(!projected.is_complete_bucket_usage_snapshot());
|
||||
assert!(!projected.is_valid_partial_snapshot());
|
||||
|
||||
projected.buckets_usage.clear();
|
||||
projected.bucket_sizes.clear();
|
||||
projected.calculate_totals();
|
||||
projected.buckets_count = 0;
|
||||
let unknown = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(projected, ["recreated".to_string()]);
|
||||
assert!(unknown.buckets_usage.is_empty());
|
||||
assert!(unknown.usage_snapshot_admin_projection);
|
||||
assert_eq!(unknown.usage_snapshot_converged, Some(false));
|
||||
assert!(!unknown.is_complete_bucket_usage_snapshot());
|
||||
assert_eq!(unknown.last_update, Some(std::time::SystemTime::UNIX_EPOCH));
|
||||
}
|
||||
|
||||
/// A write into a bucket without a verified baseline leaves process-local
|
||||
/// counts the overlay cannot reconcile, so it clears
|
||||
/// `usage_snapshot_complete`. That must flag the response non-converged,
|
||||
@@ -1048,15 +1009,11 @@ mod tests {
|
||||
|
||||
crate::app::storage_api::test::data_usage::record_bucket_object_write_memory(unverified_bucket, None, 1_024).await;
|
||||
|
||||
// Exercise the production overlay, followed by the same identity-bound
|
||||
// presentation function used by the admin query.
|
||||
crate::app::storage_api::test::data_usage::apply_bucket_usage_memory_overlay(&mut persisted).await;
|
||||
persisted.usage_snapshot_admin_projection = true;
|
||||
persisted.usage_snapshot_converged = Some(false);
|
||||
let info = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(
|
||||
let info = DefaultAdminUsecase::present_data_usage_snapshot(
|
||||
persisted,
|
||||
[measured_bucket.to_string(), unverified_bucket.to_string()],
|
||||
);
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(
|
||||
info.buckets_usage
|
||||
|
||||
@@ -22,7 +22,7 @@ use super::storage_api::bucket_usecase::StorageObjectInfo as ObjectInfo;
|
||||
#[cfg(test)]
|
||||
use super::storage_api::bucket_usecase::access::ReqInfo;
|
||||
use super::storage_api::bucket_usecase::access::{
|
||||
TableDataPlaneListAccess, authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
|
||||
authorize_request, bucket_config_mutation_incarnation, log_list_buckets_iam_implicit_deny,
|
||||
prepare_list_buckets_iam_authorization, prepare_odm_read_generation, req_info_ref,
|
||||
};
|
||||
#[cfg(test)]
|
||||
@@ -121,7 +121,6 @@ use tracing::{Instrument as _, debug, error, info, instrument, warn};
|
||||
mod accelerate;
|
||||
mod acl;
|
||||
mod core;
|
||||
#[cfg(any(test, feature = "gateway-stack"))]
|
||||
pub(crate) use self::core::requested_location_constraint;
|
||||
mod cors;
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -22,7 +22,9 @@ use crate::app::storage_api::s3::dto::{
|
||||
UserMetadata,
|
||||
};
|
||||
use crate::app::storage_api::s3::xml::{SerResult, SerializeContent};
|
||||
use crate::app::storage_api::s3::{S3Request, S3Response, S3Result, xml};
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::{S3ErrorCode, S3Request, S3Response};
|
||||
use crate::app::storage_api::s3::{S3Result, xml};
|
||||
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
|
||||
@@ -320,31 +322,20 @@ fn metadata_count_to_i32(value: usize) -> i32 {
|
||||
i32::try_from(value).unwrap_or(i32::MAX)
|
||||
}
|
||||
|
||||
async fn is_list_objects_metadata_action_allowed<T>(
|
||||
req: &S3Request<T>,
|
||||
async fn is_list_objects_metadata_action_allowed(
|
||||
envelope: &RequestEnvelope,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
action: S3Action,
|
||||
) -> S3Result<bool> {
|
||||
let mut auth_req = S3Request {
|
||||
input: (),
|
||||
method: req.method.clone(),
|
||||
uri: req.uri.clone(),
|
||||
headers: req.headers.clone(),
|
||||
extensions: req.extensions.clone(),
|
||||
credentials: req.credentials.clone(),
|
||||
region: req.region.clone(),
|
||||
service: req.service.clone(),
|
||||
trailing_headers: req.trailing_headers.clone(),
|
||||
};
|
||||
|
||||
let mut req_info = req_info_ref(req)?.clone();
|
||||
let mut auth_req = envelope.fork_for_authorization();
|
||||
let mut req_info = req_info_ref(envelope)?.clone();
|
||||
req_info.bucket = Some(bucket.to_string());
|
||||
req_info.object = Some(object.to_string());
|
||||
req_info.version_id = None;
|
||||
// Denial here is an expected filter outcome, not an error (issue #5740).
|
||||
req_info.suppress_denial_log = true;
|
||||
auth_req.extensions.insert(req_info);
|
||||
auth_req.install_req_info(req_info);
|
||||
|
||||
match authorize_request(&mut auth_req, Action::S3Action(action)).await {
|
||||
Ok(()) => Ok(true),
|
||||
@@ -353,8 +344,8 @@ async fn is_list_objects_metadata_action_allowed<T>(
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_list_objects_metadata_permissions<T>(
|
||||
req: &S3Request<T>,
|
||||
async fn collect_list_objects_metadata_permissions(
|
||||
envelope: &RequestEnvelope,
|
||||
bucket: &str,
|
||||
objects: &[ObjectInfo],
|
||||
) -> S3Result<HashMap<String, ObjectMetadataPermissions>> {
|
||||
@@ -366,9 +357,9 @@ async fn collect_list_objects_metadata_permissions<T>(
|
||||
}
|
||||
|
||||
let metadata_allowed =
|
||||
is_list_objects_metadata_action_allowed(req, bucket, &object.name, S3Action::GetObjectAction).await?;
|
||||
is_list_objects_metadata_action_allowed(envelope, bucket, &object.name, S3Action::GetObjectAction).await?;
|
||||
let tags_allowed =
|
||||
is_list_objects_metadata_action_allowed(req, bucket, &object.name, S3Action::GetObjectTaggingAction).await?;
|
||||
is_list_objects_metadata_action_allowed(envelope, bucket, &object.name, S3Action::GetObjectTaggingAction).await?;
|
||||
|
||||
permissions.insert(
|
||||
object.name.clone(),
|
||||
@@ -605,11 +596,22 @@ fn build_list_objects_v2_metadata_output(
|
||||
}
|
||||
|
||||
impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn execute_list_objects_v2m(
|
||||
&self,
|
||||
req: S3Request<ListObjectsV2Input>,
|
||||
) -> S3Result<S3Response<ListObjectsV2MetadataOutput>> {
|
||||
let input = req.input.clone();
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
self.list_objects_v2_metadata(&envelope, req.input).await.map(S3Response::new)
|
||||
}
|
||||
|
||||
/// One page of `GET /{bucket}?list-type=2&metadata=true`, for the caller the envelope's
|
||||
/// authorization installed: each entry's metadata and tags as far as that caller may read them.
|
||||
pub(crate) async fn list_objects_v2_metadata(
|
||||
&self,
|
||||
envelope: &RequestEnvelope,
|
||||
input: ListObjectsV2Input,
|
||||
) -> S3Result<ListObjectsV2MetadataOutput> {
|
||||
let ListObjectsV2Input {
|
||||
bucket,
|
||||
continuation_token,
|
||||
@@ -624,14 +626,14 @@ impl DefaultBucketUsecase {
|
||||
|
||||
let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
|
||||
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), req.uri.query())?;
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
let incl_deleted = get_header(&req.headers, rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
.map(|value| value.as_ref() == "true")
|
||||
.unwrap_or_default();
|
||||
|
||||
let object_infos = match req.extensions.get::<TableDataPlaneListAccess>() {
|
||||
let object_infos = match envelope.table_list_access() {
|
||||
Some(access) => {
|
||||
table_list_isolation::list_objects_v2(
|
||||
store.clone(),
|
||||
@@ -664,24 +666,34 @@ impl DefaultBucketUsecase {
|
||||
.map_err(ApiError::from)?,
|
||||
};
|
||||
|
||||
let permissions = collect_list_objects_metadata_permissions(&req, &bucket, &object_infos.objects).await?;
|
||||
let output = build_list_objects_v2_metadata_output(
|
||||
let permissions = collect_list_objects_metadata_permissions(envelope, &bucket, &object_infos.objects).await?;
|
||||
Ok(build_list_objects_v2_metadata_output(
|
||||
object_infos,
|
||||
&bucket,
|
||||
¶ms,
|
||||
encoding_type.as_ref(),
|
||||
fetch_owner.unwrap_or_default(),
|
||||
&permissions,
|
||||
);
|
||||
|
||||
Ok(S3Response::new(output))
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn execute_list_object_versions_m(
|
||||
&self,
|
||||
req: S3Request<ListObjectVersionsInput>,
|
||||
) -> S3Result<S3Response<ListObjectVersionsMetadataOutput>> {
|
||||
let input = req.input.clone();
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
self.list_object_versions_metadata(&envelope, req.input)
|
||||
.await
|
||||
.map(S3Response::new)
|
||||
}
|
||||
|
||||
/// One page of `GET /{bucket}?versions&metadata=true`, as [`Self::list_objects_v2_metadata`].
|
||||
pub(crate) async fn list_object_versions_metadata(
|
||||
&self,
|
||||
envelope: &RequestEnvelope,
|
||||
input: ListObjectVersionsInput,
|
||||
) -> S3Result<ListObjectVersionsMetadataOutput> {
|
||||
let ListObjectVersionsInput {
|
||||
bucket,
|
||||
delimiter,
|
||||
@@ -696,7 +708,7 @@ impl DefaultBucketUsecase {
|
||||
let params = parse_list_object_versions_params(prefix, delimiter, key_marker, version_id_marker, max_keys)?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
let object_infos = match req.extensions.get::<TableDataPlaneListAccess>() {
|
||||
let object_infos = match envelope.table_list_access() {
|
||||
Some(access) => {
|
||||
table_list_isolation::list_object_versions(
|
||||
store.clone(),
|
||||
@@ -725,11 +737,14 @@ impl DefaultBucketUsecase {
|
||||
.map_err(ApiError::from)?,
|
||||
};
|
||||
|
||||
let permissions = collect_list_objects_metadata_permissions(&req, &bucket, &object_infos.objects).await?;
|
||||
let output =
|
||||
build_list_object_versions_metadata_output(object_infos, &bucket, ¶ms, encoding_type.as_ref(), &permissions);
|
||||
|
||||
Ok(S3Response::new(output))
|
||||
let permissions = collect_list_objects_metadata_permissions(envelope, &bucket, &object_infos.objects).await?;
|
||||
Ok(build_list_object_versions_metadata_output(
|
||||
object_infos,
|
||||
&bucket,
|
||||
¶ms,
|
||||
encoding_type.as_ref(),
|
||||
&permissions,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user