mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 20:36:38 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7d2dffe138 | |||
| cbeb191e99 | |||
| ed424e2712 |
@@ -1,20 +0,0 @@
|
||||
# Report-only calibration baseline from https://github.com/rustfs/rustfs/actions/runs/29394996173.
|
||||
# Update counts only with a linked coverage run and a reviewed explanation.
|
||||
phase = "report-only"
|
||||
allowed_drop_percentage_points = 1.0
|
||||
|
||||
[crates."crates/iam"]
|
||||
covered = 5149
|
||||
count = 8131
|
||||
|
||||
[crates."crates/kms"]
|
||||
covered = 2950
|
||||
count = 4200
|
||||
|
||||
[crates."crates/policy"]
|
||||
covered = 4636
|
||||
count = 5464
|
||||
|
||||
[crates."crates/crypto"]
|
||||
covered = 469
|
||||
count = 494
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
|
||||
sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
|
||||
sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
|
||||
sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
|
||||
|
||||
@@ -36,8 +36,6 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_manual_transition_runbooks.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_security_coverage.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/s3-tests/test_report_compat.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
|
||||
@@ -14,10 +14,9 @@
|
||||
|
||||
name: "Schedule Failure Issue"
|
||||
description: >-
|
||||
Open (or update) a tracking issue when a scheduled workflow run fails or
|
||||
does not complete normally.
|
||||
Open (or update) a tracking issue when a scheduled workflow run fails.
|
||||
Dedupes by workflow name: if an open issue titled
|
||||
"[scheduled-failure] <workflow name>" already exists, the result is
|
||||
"[scheduled-failure] <workflow name>" already exists, the failure is
|
||||
appended as a comment; otherwise a new issue is created. This is the
|
||||
single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
|
||||
|
||||
@@ -39,30 +38,6 @@ inputs:
|
||||
Set to an empty string to skip labeling.
|
||||
required: false
|
||||
default: "infrastructure"
|
||||
source-run-id:
|
||||
description: "Run ID to report. Defaults to the current workflow run."
|
||||
required: false
|
||||
default: ${{ github.run_id }}
|
||||
source-run-attempt:
|
||||
description: "Run attempt to report. Defaults to the current attempt."
|
||||
required: false
|
||||
default: ${{ github.run_attempt }}
|
||||
source-event:
|
||||
description: "Trigger event of the run being reported."
|
||||
required: false
|
||||
default: ${{ github.event_name }}
|
||||
source-ref-name:
|
||||
description: "Ref name of the run being reported."
|
||||
required: false
|
||||
default: ${{ github.ref_name }}
|
||||
source-sha:
|
||||
description: "Commit SHA of the run being reported."
|
||||
required: false
|
||||
default: ${{ github.sha }}
|
||||
details-file:
|
||||
description: "Optional Markdown file appended to the issue body."
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
@@ -73,22 +48,17 @@ runs:
|
||||
GH_TOKEN: ${{ inputs.github-token }}
|
||||
WORKFLOW_NAME: ${{ inputs.workflow-name }}
|
||||
ISSUE_LABEL: ${{ inputs.label }}
|
||||
SOURCE_RUN_ID: ${{ inputs.source-run-id }}
|
||||
SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
|
||||
SOURCE_EVENT: ${{ inputs.source-event }}
|
||||
SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
|
||||
SOURCE_SHA: ${{ inputs.source-sha }}
|
||||
DETAILS_FILE: ${{ inputs.details-file }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
title="[scheduled-failure] ${WORKFLOW_NAME}"
|
||||
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
|
||||
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
|
||||
# Inspect the reported run attempt. It can be the current in-workflow
|
||||
# failure or a completed run observed by the external watchdog.
|
||||
# Failed job names for this run attempt. The alert job runs while the
|
||||
# run as a whole is still in progress, so inspect the jobs that have
|
||||
# already completed with a non-success conclusion.
|
||||
failed_jobs="$(gh api \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
|
||||
--paginate \
|
||||
--jq '.jobs[]
|
||||
| select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
|
||||
@@ -97,26 +67,15 @@ runs:
|
||||
failed_jobs="- (failed job not recorded yet — see the run page)"
|
||||
fi
|
||||
|
||||
details=""
|
||||
if [ -n "${DETAILS_FILE}" ]; then
|
||||
if [ -f "${DETAILS_FILE}" ]; then
|
||||
details="$(cat "${DETAILS_FILE}")"
|
||||
else
|
||||
details="Details file was not available: \`${DETAILS_FILE}\`"
|
||||
fi
|
||||
fi
|
||||
|
||||
body="$(cat <<EOF
|
||||
Run of **${WORKFLOW_NAME}** did not complete successfully.
|
||||
Scheduled run of **${WORKFLOW_NAME}** failed.
|
||||
|
||||
- Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
|
||||
- Event: \`${SOURCE_EVENT}\`
|
||||
- Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
|
||||
- Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
|
||||
- Event: \`${GITHUB_EVENT_NAME}\`
|
||||
- Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
|
||||
|
||||
Non-success jobs:
|
||||
Failed jobs:
|
||||
${failed_jobs}
|
||||
|
||||
${details}
|
||||
EOF
|
||||
)"
|
||||
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
[
|
||||
{ "workflow": ".github/workflows/audit.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/build.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
|
||||
]
|
||||
@@ -46,7 +46,7 @@ on:
|
||||
# advisory could sit unnoticed for seven days. The check list is unchanged —
|
||||
# splitting it into a light daily advisories-only run and a weekly full run
|
||||
# would create runs where sources/bans/licenses go unverified.
|
||||
- cron: '23 3 * * *' # Daily 03:23 UTC
|
||||
- cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -52,7 +52,7 @@ on:
|
||||
- ".dockerignore"
|
||||
- "flake.lock"
|
||||
schedule:
|
||||
- cron: "13 1 * * 0" # Weekly on Sunday 01:13 UTC
|
||||
- cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_docker:
|
||||
@@ -1032,23 +1032,3 @@ jobs:
|
||||
|
||||
echo "🎉 Released $TAG successfully!"
|
||||
echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -94,9 +94,6 @@ concurrency:
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
# Swatinem/rust-cache hashes every RUST* variable. Keep this aligned with
|
||||
# ci.yml or the writer and readers use disjoint cache keys.
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
|
||||
@@ -104,7 +101,7 @@ jobs:
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 120
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -194,7 +191,7 @@ jobs:
|
||||
warm-ci-feat-rio:
|
||||
name: Warm ci-feat-rio
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 120
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -222,7 +219,7 @@ jobs:
|
||||
warm-ci-feat-proto:
|
||||
name: Warm ci-feat-proto
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 120
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
|
||||
@@ -126,10 +126,7 @@ jobs:
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
@@ -59,7 +59,7 @@ on:
|
||||
merge_group:
|
||||
types: [ checks_requested ]
|
||||
schedule:
|
||||
- cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC
|
||||
- cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -161,10 +161,7 @@ jobs:
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
@@ -1035,37 +1032,3 @@ jobs:
|
||||
path: artifacts/s3tests-single/**
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs:
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- test-ilm-integration-serial
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
- e2e-full
|
||||
- e2e-tests-rio-v2
|
||||
- s3-implemented-tests
|
||||
- s3-lifecycle-behavior-tests
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -12,12 +12,14 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Workspace line-coverage baseline and security-crate calibration
|
||||
# (backlog#1153 infra-5/infra-6).
|
||||
# Weekly workspace line-coverage baseline (backlog#1153 infra-5).
|
||||
#
|
||||
# NON-BLOCKING by design: the weekly job gives coverage a visible baseline and
|
||||
# trend, while relevant pull requests run a report-only security-crate
|
||||
# comparison. Neither job is a required check during calibration.
|
||||
# NON-BLOCKING by design: this workflow only runs on schedule and manual
|
||||
# dispatch, so it never attaches a status to a PR and must never be made a
|
||||
# required check. It exists to give coverage a visible baseline and trend
|
||||
# (per-crate table in the job summary, lcov artifact kept 90 days) — the
|
||||
# per-crate ratchet for the security-critical crates builds on it later
|
||||
# (backlog#1153 infra-6, report-only first per the ci-11 ladder).
|
||||
#
|
||||
# Measurement scope matches the PR test gate (ci.yml "Run tests"):
|
||||
# `--workspace --exclude e2e_test` with the `ci` nextest profile. Doctests are
|
||||
@@ -29,28 +31,13 @@
|
||||
name: coverage
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "crates/iam/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/policy/**"
|
||||
- "crates/crypto/**"
|
||||
- ".config/coverage-baselines.toml"
|
||||
- "scripts/coverage_per_crate.py"
|
||||
- "scripts/check_security_coverage.py"
|
||||
- ".github/workflows/coverage.yml"
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# 07:00 UTC Sunday — staggered clear of the other Sunday crons: ci (00:00),
|
||||
# build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
|
||||
# (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
|
||||
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
|
||||
- cron: "43 7 * * 0"
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'schedule' }}
|
||||
- cron: "0 7 * * 0"
|
||||
|
||||
# Only alert-on-failure needs more than read access; it declares its own
|
||||
# job-level `issues: write`.
|
||||
@@ -59,13 +46,12 @@ permissions:
|
||||
|
||||
jobs:
|
||||
coverage:
|
||||
name: Workspace line coverage
|
||||
name: Workspace coverage (weekly)
|
||||
runs-on: sm-standard-4
|
||||
# The instrumented build cannot reuse the regular CI cache (different
|
||||
# RUSTFLAGS), so a cold run rebuilds the workspace before running the
|
||||
# full suite. Exact-head run 32573798257 needed 119m42s including reports
|
||||
# and artifact upload, so keep a bounded 30-minute publication margin.
|
||||
timeout-minutes: 150
|
||||
# RUSTFLAGS), so a cold week rebuilds the workspace before running the
|
||||
# full suite; give it double the test job's 60-minute budget.
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
# Match the PR gate's nextest semantics (ci.yml runs `--profile ci`):
|
||||
@@ -105,9 +91,7 @@ jobs:
|
||||
cargo llvm-cov report --json --output-path target/llvm-cov/coverage.json
|
||||
|
||||
- name: Write per-crate summary
|
||||
run: |
|
||||
python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 scripts/check_security_coverage.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
|
||||
run: python3 scripts/coverage_per_crate.py target/llvm-cov/coverage.json >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload coverage artifact
|
||||
if: always()
|
||||
|
||||
@@ -40,7 +40,7 @@ on:
|
||||
schedule:
|
||||
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
|
||||
# stale (01:30) and performance-ab (06:00).
|
||||
- cron: "29 4 * * *"
|
||||
- cron: "0 4 * * *"
|
||||
|
||||
# Only alert-on-failure needs more than read access; it declares its own
|
||||
# job-level `issues: write`.
|
||||
@@ -196,9 +196,6 @@ jobs:
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Verify protocol socket oracle
|
||||
run: ss -tn state CLOSE-WAIT >/dev/null
|
||||
|
||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
||||
- name: Verify protocol e2e membership
|
||||
env:
|
||||
|
||||
@@ -97,7 +97,7 @@ on:
|
||||
schedule:
|
||||
# Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
|
||||
# single-node and the 4-node distributed topologies (matrix below).
|
||||
- cron: "19 2 * * 0"
|
||||
- cron: "0 2 * * 0"
|
||||
|
||||
env:
|
||||
# main user
|
||||
@@ -287,7 +287,6 @@ jobs:
|
||||
|
||||
frontend fe_s3
|
||||
bind *:9000
|
||||
option http-buffer-request
|
||||
default_backend be_s3
|
||||
|
||||
backend be_s3
|
||||
|
||||
@@ -30,7 +30,7 @@ on:
|
||||
- "Cargo.lock"
|
||||
- ".github/workflows/fuzz.yml"
|
||||
schedule:
|
||||
- cron: "17 2 * * *"
|
||||
- cron: "0 2 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
profile:
|
||||
|
||||
@@ -121,21 +121,3 @@ jobs:
|
||||
cargo nextest run --run-ignored ignored-only --no-tests=fail \
|
||||
-p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
|
||||
-E "$INTEROP_FILTER"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [minio-interop]
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -45,6 +45,13 @@
|
||||
# docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
|
||||
# has fewer cores and reintroduces fleet-state risk for no reliability gain.
|
||||
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||
# reading this file, which has already misled at least one audit — hence this
|
||||
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||
#
|
||||
name: mint
|
||||
|
||||
on:
|
||||
@@ -63,13 +70,13 @@ on:
|
||||
- core
|
||||
- full
|
||||
mint-image:
|
||||
description: "Mint image reference (empty = pinned default)"
|
||||
description: "Mint image reference"
|
||||
required: false
|
||||
default: ""
|
||||
default: "minio/mint:edge"
|
||||
schedule:
|
||||
# Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to
|
||||
# 3h) has finished, so the two never contend for the same runner pool.
|
||||
- cron: "41 6 * * 0"
|
||||
- cron: "0 6 * * 0"
|
||||
|
||||
env:
|
||||
S3_ACCESS_KEY: rustfsadmin-ci
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Nightly GNU Build
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "7 0 * * *"
|
||||
- cron: "0 0 * * *"
|
||||
timezone: "Asia/Shanghai"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -194,23 +194,3 @@ jobs:
|
||||
|
||||
- name: Run HA leader failover live checks (three-node Raft cluster in Docker)
|
||||
run: bash scripts/test/vault_ha_kms_live.sh
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [build, kms-vault-lane, kms-vault-ha-failover]
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -22,15 +22,22 @@
|
||||
# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
|
||||
# blocked (rustfs/backlog#935 correction 1).
|
||||
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||
# reading this file, which has already misled at least one audit — hence this
|
||||
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||
#
|
||||
name: Performance A/B
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "31 6 * * *" # 06:31 UTC nightly, against main
|
||||
- cron: "0 6 * * *" # 06:00 UTC nightly, against main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
duration:
|
||||
description: "warp duration per round"
|
||||
description: "warp duration per round (short by default to fit the double-build budget)"
|
||||
required: false
|
||||
default: "12s"
|
||||
type: string
|
||||
@@ -39,8 +46,12 @@ on:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
push:
|
||||
# Every main commit pre-builds and caches its release binary (perf-3) so the
|
||||
# nightly A/B restores a ready baseline instead of paying the double build.
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
env:
|
||||
@@ -48,19 +59,83 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
warp-ab:
|
||||
name: Warp A/B budget gate
|
||||
# perf-3: on every push to main, build the release binary once and cache it
|
||||
# keyed by commit SHA (rustfs-baseline-<sha>). The warp-ab measurements
|
||||
# restore this instead of paying the ~32min-per-side source
|
||||
# build. That double build is what pushed the expanded 24-cell nightly past its
|
||||
# ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
|
||||
# builds off the shared cargo cache keep each push cheap, and building on the
|
||||
# same sm-standard-2 runner the A/B measures on guarantees the cached binary is
|
||||
# ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
|
||||
# are cancelled ~7/8 of the time and are not a reliable baseline.
|
||||
build-baseline-cache:
|
||||
name: Build + cache baseline binary
|
||||
if: github.event_name == 'push'
|
||||
runs-on: sm-standard-2
|
||||
# A normal nightly restores the last successful binary and builds only the
|
||||
# candidate; daily access keeps that cache warm. A cache miss may build both
|
||||
# and needs room for the A/B run plus artifact and cache publication.
|
||||
timeout-minutes: 180
|
||||
# Latest-wins: consumers only ever restore the binary for the *current*
|
||||
# origin/main tip, so when pushes land faster than the ~65min build, a
|
||||
# superseded build's output is dead weight — cancel it instead of stacking
|
||||
# hour-long jobs on the shared runner pool. A skipped intermediate SHA at
|
||||
# most costs one same-commit self-heal in the A/B job.
|
||||
concurrency:
|
||||
group: perf-baseline-build-main
|
||||
cancel-in-progress: true
|
||||
# #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
|
||||
# single release build past 60min on this runner — every cache build on
|
||||
# 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
|
||||
# time of 1h0m0s") and the cache never populated. The measured binary must
|
||||
# keep the production profile, so the budget absorbs the build instead.
|
||||
timeout-minutes: 100
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0 # baseline may be an earlier successful scheduled head
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build release rustfs
|
||||
run: cargo build --release --bin rustfs
|
||||
|
||||
- name: Stage binary for cache
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p baseline-bin
|
||||
cp target/release/rustfs baseline-bin/rustfs
|
||||
|
||||
- name: Cache baseline binary by SHA
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ github.sha }}
|
||||
|
||||
warp-ab:
|
||||
name: Warp A/B budget gate
|
||||
# Always run on schedule / manual dispatch. Never on push — that event only
|
||||
# feeds build-baseline-cache above.
|
||||
if: >-
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
runs-on: sm-standard-2
|
||||
# With perf-3's cached baseline binary the common (cache-hit) nightly is
|
||||
# measurement-only and finishes well under 50min. This ceiling stays
|
||||
# generous only to absorb the same-commit cache-miss self-heal (~65min
|
||||
# single build with the post-#4806 LTO profile + measurement). A timeout
|
||||
# surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
|
||||
# not just failure). perf-6 recalibrates the budget once the noise study
|
||||
# lands.
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0 # baseline is built from origin/main
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -88,55 +163,24 @@ jobs:
|
||||
fi
|
||||
echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# A failed regression run must keep comparing against the last known-good
|
||||
# scheduled head. Otherwise the next nightly would absorb the regression
|
||||
# into its baseline and turn green without a fix.
|
||||
- name: Find last successful scheduled baseline
|
||||
id: scheduled_baseline
|
||||
if: github.event_name == 'schedule'
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const { data } = await github.rest.actions.listWorkflowRuns({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
workflow_id: "performance-ab.yml",
|
||||
event: "schedule",
|
||||
status: "success",
|
||||
per_page: 1,
|
||||
});
|
||||
return data.workflow_runs[0]?.head_sha ?? "";
|
||||
|
||||
# Manual runs compare a selected ref with current main. Scheduled runs
|
||||
# compare current main with the last successful scheduled head. With no
|
||||
# history, the first run measures the candidate against itself and seeds
|
||||
# that head only if the complete rig succeeds.
|
||||
# perf-3: resolve the commits so the cache can be keyed by SHA. The
|
||||
# baseline is origin/main; the candidate is the checked-out ref. On the
|
||||
# nightly (checkout == main) they are the same commit, so one cached binary
|
||||
# serves both phases and the run does zero source builds.
|
||||
- name: Resolve baseline / candidate commits
|
||||
id: commits
|
||||
env:
|
||||
SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
baseline_sha="$(git rev-parse origin/main)"
|
||||
candidate_sha="$(git rev-parse HEAD)"
|
||||
if [[ "${{ github.event_name }}" == "schedule" ]]; then
|
||||
baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
|
||||
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
|
||||
echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
baseline_sha="$(git rev-parse origin/main)"
|
||||
fi
|
||||
git cat-file -e "${baseline_sha}^{commit}"
|
||||
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline commit: $baseline_sha"
|
||||
echo "candidate commit: $candidate_sha"
|
||||
|
||||
# Exact-key restore of the candidate binary saved by its successful
|
||||
# scheduled run. A miss leaves cache-hit unset and falls back to a source
|
||||
# build of that known-good head.
|
||||
# Exact-key restore of the baseline binary built by build-baseline-cache
|
||||
# when origin/main last landed. A miss (binary evicted or not built yet)
|
||||
# leaves cache-hit unset and the rig falls back to a source build.
|
||||
- name: Restore cached baseline binary
|
||||
id: baseline_cache
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
@@ -226,11 +270,11 @@ jobs:
|
||||
elif [[ "$selfheal_built" == "true" ]]; then
|
||||
base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
|
||||
else
|
||||
base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
|
||||
base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)"
|
||||
fi
|
||||
if [[ "$candidate_sha" == "$baseline_sha" ]]; then
|
||||
# No commits landed since the last successful baseline, so reuse
|
||||
# the one binary for both phases and measure only rig drift.
|
||||
# Nightly on main: the candidate is the same commit as the baseline,
|
||||
# so reuse the one binary for both phases and skip all builds.
|
||||
args+=(--candidate-bin "$base_bin")
|
||||
cand_src="same binary as baseline (same commit)"
|
||||
elif [[ "$candidate_built" == "true" ]]; then
|
||||
@@ -318,23 +362,6 @@ jobs:
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stage successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cp candidate-bin/rustfs baseline-bin/rustfs
|
||||
|
||||
- name: Cache successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
|
||||
|
||||
# Scheduled failure alerting is handled by the alert-on-failure job below
|
||||
# (perf-2 consuming ci-8's schedule-failure-issue composite action).
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ name: Runner Hygiene
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC
|
||||
- cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Scheduled Validation Freshness
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "47 23 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: scheduled-validation-freshness
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
check-freshness:
|
||||
name: Check scheduled validation freshness
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check latest scheduled runs
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
python3 scripts/check_scheduled_validation_freshness.py \
|
||||
--report "${RUNNER_TEMP}/scheduled-validation-freshness.md"
|
||||
status=$?
|
||||
cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
exit "${status}"
|
||||
- name: Open or update freshness issue
|
||||
if: failure()
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
details-file: ${{ runner.temp }}/scheduled-validation-freshness.md
|
||||
@@ -1,63 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Scheduled Validation Watchdog
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows:
|
||||
- "Security Audit"
|
||||
- "Build and Release"
|
||||
- "Continuous Integration"
|
||||
- "coverage"
|
||||
- "e2e-nightly"
|
||||
- "e2e-s3tests"
|
||||
- "Fuzz"
|
||||
- "mint"
|
||||
- "minio-interop"
|
||||
- "Nightly GNU Build"
|
||||
- "Performance A/B"
|
||||
- "Runner Hygiene"
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
alert-on-incomplete-run:
|
||||
name: Alert on incomplete scheduled run
|
||||
if: >-
|
||||
github.event.workflow_run.event == 'schedule' &&
|
||||
github.event.workflow_run.conclusion != 'success' &&
|
||||
github.event.workflow_run.conclusion != 'failure'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update incomplete-run issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
workflow-name: ${{ github.event.workflow_run.name }}
|
||||
source-run-id: ${{ github.event.workflow_run.id }}
|
||||
source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
|
||||
source-event: ${{ github.event.workflow_run.event }}
|
||||
source-ref-name: ${{ github.event.workflow_run.head_branch }}
|
||||
source-sha: ${{ github.event.workflow_run.head_sha }}
|
||||
@@ -901,10 +901,6 @@ pub struct Metrics {
|
||||
scanner_cycle_max_duration_millis: AtomicU64,
|
||||
scanner_cycle_max_objects: AtomicU64,
|
||||
scanner_cycle_max_directories: AtomicU64,
|
||||
scanner_cycle_timeout_total: AtomicU64,
|
||||
scanner_cycle_recovery_required_total: AtomicU64,
|
||||
scanner_cycle_last_progress_age_seconds: AtomicU64,
|
||||
scanner_leader_lease_without_progress: AtomicBool,
|
||||
scanner_bitrot_cycle_enabled: AtomicBool,
|
||||
scanner_bitrot_cycle_millis: AtomicU64,
|
||||
scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
|
||||
@@ -1374,14 +1370,6 @@ pub struct ScannerMetricsReport {
|
||||
#[serde(default)]
|
||||
pub cycle_max_directories: u64,
|
||||
#[serde(default)]
|
||||
pub cycle_timeout_total: u64,
|
||||
#[serde(default)]
|
||||
pub cycle_recovery_required_total: u64,
|
||||
#[serde(default)]
|
||||
pub cycle_last_progress_age: u64,
|
||||
#[serde(default)]
|
||||
pub leader_lease_without_progress: bool,
|
||||
#[serde(default)]
|
||||
pub bitrot_cycle_enabled: bool,
|
||||
#[serde(default)]
|
||||
pub bitrot_cycle_seconds: f64,
|
||||
@@ -1442,9 +1430,6 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total
|
||||
const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
|
||||
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
|
||||
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
|
||||
const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
|
||||
const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
|
||||
const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
|
||||
|
||||
fn scan_cycle_result_label(result: u8) -> &'static str {
|
||||
match result {
|
||||
@@ -1928,10 +1913,6 @@ impl Metrics {
|
||||
scanner_cycle_max_duration_millis: AtomicU64::new(0),
|
||||
scanner_cycle_max_objects: AtomicU64::new(0),
|
||||
scanner_cycle_max_directories: AtomicU64::new(0),
|
||||
scanner_cycle_timeout_total: AtomicU64::new(0),
|
||||
scanner_cycle_recovery_required_total: AtomicU64::new(0),
|
||||
scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
|
||||
scanner_leader_lease_without_progress: AtomicBool::new(false),
|
||||
scanner_bitrot_cycle_enabled: AtomicBool::new(false),
|
||||
scanner_bitrot_cycle_millis: AtomicU64::new(0),
|
||||
scanner_checkpoint: Mutex::new(None),
|
||||
@@ -2431,29 +2412,12 @@ impl Metrics {
|
||||
.store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
|
||||
self.scanner_cycle_max_directories
|
||||
.store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
|
||||
self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
|
||||
self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
|
||||
metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
|
||||
metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
|
||||
self.scanner_bitrot_cycle_enabled
|
||||
.store(bitrot_cycle.is_some(), Ordering::Relaxed);
|
||||
self.scanner_bitrot_cycle_millis
|
||||
.store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
|
||||
self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
|
||||
if recovery_required {
|
||||
self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.scanner_cycle_last_progress_age_seconds
|
||||
.store(progress_age.as_secs(), Ordering::Relaxed);
|
||||
self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
|
||||
metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
|
||||
metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
|
||||
metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
|
||||
}
|
||||
|
||||
pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
|
||||
if let Some(concurrency_limit) = concurrency_limit {
|
||||
self.scanner_set_scan_concurrency_limit
|
||||
@@ -3301,10 +3265,6 @@ impl Metrics {
|
||||
m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
|
||||
m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
|
||||
m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
|
||||
m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
|
||||
m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
|
||||
m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
|
||||
m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
|
||||
m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.scan_checkpoint = match self.scanner_checkpoint.lock() {
|
||||
@@ -4966,20 +4926,4 @@ mod tests {
|
||||
assert!(!report.bitrot_cycle_enabled);
|
||||
assert_eq!(report.bitrot_cycle_seconds, 0.0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
|
||||
let timed_out = metrics.report().await;
|
||||
assert_eq!(timed_out.cycle_timeout_total, 1);
|
||||
assert_eq!(timed_out.cycle_last_progress_age, 17);
|
||||
assert!(timed_out.leader_lease_without_progress);
|
||||
|
||||
metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
|
||||
let current = metrics.report().await;
|
||||
assert_eq!(current.cycle_timeout_total, 1);
|
||||
assert_eq!(current.cycle_last_progress_age, 0);
|
||||
assert!(!current.leader_lease_without_progress);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,12 +84,6 @@ Current guidance:
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
|
||||
|
||||
Scanner cycle budget controls:
|
||||
|
||||
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
|
||||
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
|
||||
- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
|
||||
|
||||
## Mmap read environment aliases
|
||||
|
||||
- `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical)
|
||||
|
||||
@@ -143,12 +143,9 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
|
||||
/// Default scanner speed preset.
|
||||
pub const DEFAULT_SCANNER_SPEED: &str = "default";
|
||||
|
||||
/// Default scanner cycle runtime budget when no override is configured.
|
||||
///
|
||||
/// An explicit `0` remains the compatibility escape hatch for an unbounded
|
||||
/// cycle. Keeping the unset default finite prevents a stalled scanner I/O
|
||||
/// operation from holding the leader lease forever.
|
||||
pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60;
|
||||
/// Default scanner cycle runtime budget.
|
||||
/// `0` keeps the existing unbounded per-cycle behavior.
|
||||
pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0;
|
||||
|
||||
/// Default scanner per-cycle object budget.
|
||||
/// `0` keeps the existing unbounded per-cycle behavior.
|
||||
|
||||
@@ -585,12 +585,9 @@ impl VersionsHistogram {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication statistics for a single target.
|
||||
///
|
||||
/// Renamed from `ReplicationStats`; serde field names are preserved
|
||||
/// byte-identically to maintain wire compatibility with existing snapshots.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ReplicationTargetUsage {
|
||||
/// Replication statistics for a single target
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicationStats {
|
||||
pub pending_size: u64,
|
||||
pub replicated_size: u64,
|
||||
pub failed_size: u64,
|
||||
@@ -603,7 +600,7 @@ pub struct ReplicationTargetUsage {
|
||||
pub replicated_count: u64,
|
||||
}
|
||||
|
||||
impl ReplicationTargetUsage {
|
||||
impl ReplicationStats {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
let Self {
|
||||
pending_size,
|
||||
@@ -639,7 +636,7 @@ impl ReplicationTargetUsage {
|
||||
/// Replication statistics for all targets
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicationAllStats {
|
||||
pub targets: HashMap<String, ReplicationTargetUsage>,
|
||||
pub targets: HashMap<String, ReplicationStats>,
|
||||
pub replica_size: u64,
|
||||
pub replica_count: u64,
|
||||
}
|
||||
@@ -652,7 +649,7 @@ impl ReplicationAllStats {
|
||||
targets,
|
||||
} = self;
|
||||
|
||||
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationTargetUsage::is_empty)
|
||||
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
|
||||
}
|
||||
|
||||
#[deprecated(note = "use is_empty instead")]
|
||||
@@ -2469,7 +2466,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn replication_stats_empty_checks_every_field() {
|
||||
type SetField = fn(&mut ReplicationTargetUsage);
|
||||
type SetField = fn(&mut ReplicationStats);
|
||||
|
||||
let cases: [(&str, SetField); 10] = [
|
||||
("pending_size", |stats| stats.pending_size = 1),
|
||||
@@ -2484,9 +2481,9 @@ mod tests {
|
||||
("replicated_count", |stats| stats.replicated_count = 1),
|
||||
];
|
||||
|
||||
assert!(ReplicationTargetUsage::default().is_empty());
|
||||
assert!(ReplicationStats::default().is_empty());
|
||||
for (field, set_nonzero) in cases {
|
||||
let mut stats = ReplicationTargetUsage::default();
|
||||
let mut stats = ReplicationStats::default();
|
||||
set_nonzero(&mut stats);
|
||||
assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
|
||||
}
|
||||
@@ -2517,17 +2514,17 @@ mod tests {
|
||||
}
|
||||
|
||||
let empty_targets = ReplicationAllStats {
|
||||
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationTargetUsage::default())]),
|
||||
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
|
||||
|
||||
let stats = ReplicationAllStats {
|
||||
targets: HashMap::from([
|
||||
("arn:test:empty".to_string(), ReplicationTargetUsage::default()),
|
||||
("arn:test:empty".to_string(), ReplicationStats::default()),
|
||||
(
|
||||
"arn:test:non-empty".to_string(),
|
||||
ReplicationTargetUsage {
|
||||
ReplicationStats {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2568,7 +2565,7 @@ mod tests {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:pending".to_string(),
|
||||
ReplicationTargetUsage {
|
||||
ReplicationStats {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2717,7 +2714,7 @@ mod tests {
|
||||
targets: HashMap::from([
|
||||
(
|
||||
"arn:self-only".to_string(),
|
||||
ReplicationTargetUsage {
|
||||
ReplicationStats {
|
||||
pending_size: 7,
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
@@ -2725,7 +2722,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
"arn:shared".to_string(),
|
||||
ReplicationTargetUsage {
|
||||
ReplicationStats {
|
||||
failed_size: 3,
|
||||
failed_count: 1,
|
||||
missed_threshold_size: 2,
|
||||
@@ -2744,7 +2741,7 @@ mod tests {
|
||||
targets: HashMap::from([
|
||||
(
|
||||
"arn:shared".to_string(),
|
||||
ReplicationTargetUsage {
|
||||
ReplicationStats {
|
||||
failed_size: 5,
|
||||
failed_count: 2,
|
||||
after_threshold_size: 4,
|
||||
@@ -2754,7 +2751,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
"arn:other-only".to_string(),
|
||||
ReplicationTargetUsage {
|
||||
ReplicationStats {
|
||||
replicated_size: 11,
|
||||
replicated_count: 3,
|
||||
..Default::default()
|
||||
@@ -2996,9 +2993,7 @@ mod tests {
|
||||
fn replication_target_deserialization_preserves_large_historical_maps() {
|
||||
let mut stats = ReplicationAllStats::default();
|
||||
for index in 0..=1024 {
|
||||
stats
|
||||
.targets
|
||||
.insert(format!("target-{index}"), ReplicationTargetUsage::default());
|
||||
stats.targets.insert(format!("target-{index}"), ReplicationStats::default());
|
||||
}
|
||||
let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode");
|
||||
let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded)
|
||||
@@ -3007,47 +3002,6 @@ mod tests {
|
||||
assert_eq!(decoded.targets.len(), stats.targets.len());
|
||||
}
|
||||
|
||||
/// Round-trip test: encoding a [`ReplicationTargetUsage`] and decoding it back
|
||||
/// must produce the exact same value. This guards against accidental serde
|
||||
/// field-name drift during the `ReplicationStats` -> `ReplicationTargetUsage`
|
||||
/// rename. Wire-level field names are the serialized Rust field identifiers,
|
||||
/// which must remain byte-identical.
|
||||
#[test]
|
||||
fn replication_target_usage_rmp_round_trip() {
|
||||
let original = ReplicationTargetUsage {
|
||||
pending_size: 100,
|
||||
replicated_size: 2_000,
|
||||
failed_size: 50,
|
||||
failed_count: 3,
|
||||
pending_count: 7,
|
||||
missed_threshold_size: 11,
|
||||
after_threshold_size: 22,
|
||||
missed_threshold_count: 1,
|
||||
after_threshold_count: 2,
|
||||
replicated_count: 99,
|
||||
};
|
||||
|
||||
let buf = rmp_serde::to_vec_named(&original).expect("encode ReplicationTargetUsage to msgpack");
|
||||
let decoded: ReplicationTargetUsage = rmp_serde::from_slice(&buf).expect("decode ReplicationTargetUsage from msgpack");
|
||||
assert_eq!(original, decoded, "round-trip through rmp must preserve every field");
|
||||
|
||||
// Also verify that encoding as an unnamed sequence and then decoding
|
||||
// with named fields produces the correct mapping (this catches reordering).
|
||||
let named_buf = rmp_serde::to_vec_named(&original).expect("re-encode for field-name pinning");
|
||||
// Spot-check that known field names appear in the named encoding.
|
||||
let named_str = String::from_utf8_lossy(&named_buf);
|
||||
assert!(named_str.contains("pending_size"), "field 'pending_size' must survive the rename");
|
||||
assert!(named_str.contains("replicated_size"), "field 'replicated_size' must survive the rename");
|
||||
assert!(
|
||||
named_str.contains("missed_threshold_size"),
|
||||
"field 'missed_threshold_size' must survive the rename"
|
||||
);
|
||||
assert!(
|
||||
named_str.contains("after_threshold_count"),
|
||||
"field 'after_threshold_count' must survive the rename"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_merge_rejects_noncanonical_histograms_without_mutation() {
|
||||
let mut entry = DataUsageEntry {
|
||||
|
||||
@@ -15,12 +15,14 @@
|
||||
use crate::common::RustFSTestClusterEnvironment;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::types::{CorsConfiguration, CorsRule};
|
||||
use bytes::Bytes;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Barrier;
|
||||
use tracing::{info, warn};
|
||||
|
||||
const BUCKET: &str = "conditional-put-race-bucket";
|
||||
const BUCKET_METADATA_RELOAD_BUCKET: &str = "bucket-metadata-reload-barrier";
|
||||
|
||||
async fn cleanup_object(client: &Client, key: &str) {
|
||||
if let Err(e) = client.delete_object().bucket(BUCKET).key(key).send().await {
|
||||
@@ -28,6 +30,16 @@ async fn cleanup_object(client: &Client, key: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_bucket_cors_missing(client: &Client) {
|
||||
let result = client.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await;
|
||||
match result {
|
||||
Err(SdkError::ServiceError(error)) => {
|
||||
assert_eq!(error.err().meta().code(), Some("NoSuchCORSConfiguration"));
|
||||
}
|
||||
result => panic!("expected the peer to report a missing CORS configuration: {result:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn conditional_put(
|
||||
client: &Client,
|
||||
key: &str,
|
||||
@@ -236,3 +248,48 @@ async fn test_conditional_put_basic_cluster() -> Result<(), Box<dyn std::error::
|
||||
cleanup_object(&client, test_key).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_cors_write_is_visible_on_peer_before_response() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
crate::common::init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(BUCKET_METADATA_RELOAD_BUCKET).await?;
|
||||
|
||||
let writer = cluster.create_s3_client(0)?;
|
||||
let reader = cluster.create_s3_client(1)?;
|
||||
assert_bucket_cors_missing(&reader).await;
|
||||
|
||||
let rule = CorsRule::builder()
|
||||
.allowed_methods("GET")
|
||||
.allowed_origins("https://example.com")
|
||||
.build()?;
|
||||
let configuration = CorsConfiguration::builder().cors_rules(rule).build()?;
|
||||
|
||||
writer
|
||||
.put_bucket_cors()
|
||||
.bucket(BUCKET_METADATA_RELOAD_BUCKET)
|
||||
.cors_configuration(configuration)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let response = reader.get_bucket_cors().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
|
||||
let rules = response.cors_rules();
|
||||
assert_eq!(
|
||||
rules.len(),
|
||||
1,
|
||||
"peer should observe the committed CORS rule before the write response returns"
|
||||
);
|
||||
assert_eq!(rules[0].allowed_methods(), ["GET"]);
|
||||
assert_eq!(rules[0].allowed_origins(), ["https://example.com"]);
|
||||
|
||||
writer
|
||||
.delete_bucket_cors()
|
||||
.bucket(BUCKET_METADATA_RELOAD_BUCKET)
|
||||
.send()
|
||||
.await?;
|
||||
assert_bucket_cors_missing(&reader).await;
|
||||
writer.delete_bucket().bucket(BUCKET_METADATA_RELOAD_BUCKET).send().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -380,24 +380,10 @@ mod tests {
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let mut recovered = serde_json::Value::Null;
|
||||
for _ in 0..60 {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
assert!(
|
||||
!status_body.contains("MissingContentLength"),
|
||||
"background heal status should not fail without an explicit Content-Length: {status_body}"
|
||||
);
|
||||
recovered = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
assert_eq!(
|
||||
recovered["clusterStatusComplete"],
|
||||
serde_json::Value::Bool(true),
|
||||
"cluster heal status should recover before root heal starts: {recovered}"
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
assert!(
|
||||
!status_body.contains("MissingContentLength"),
|
||||
"background heal status should not fail without an explicit Content-Length: {status_body}"
|
||||
);
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
|
||||
@@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -159,7 +159,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -278,7 +278,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -475,7 +475,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -570,7 +570,7 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -189,121 +189,34 @@ pub async fn wait_for_kms_ready(
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await
|
||||
}
|
||||
|
||||
async fn wait_for_kms_ready_with_timeout(
|
||||
base_url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
total_deadline: Duration,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let total_deadline = Duration::from_secs(5);
|
||||
let start = tokio::time::Instant::now();
|
||||
let deadline = start + total_deadline;
|
||||
let mut backoff = Duration::from_millis(200);
|
||||
let max_backoff = Duration::from_secs(1);
|
||||
let mut first_attempt = true;
|
||||
|
||||
loop {
|
||||
match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await {
|
||||
Ok(Ok(status)) => {
|
||||
let backend_status = serde_json::from_str::<serde_json::Value>(&status)
|
||||
.ok()
|
||||
.and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned));
|
||||
if backend_status.as_deref() == Some("healthy") {
|
||||
info!("KMS is ready (status: {})", status);
|
||||
return Ok(());
|
||||
}
|
||||
warn!(
|
||||
backend_status = backend_status.as_deref().unwrap_or("missing"),
|
||||
elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
"KMS not ready yet, retrying…"
|
||||
);
|
||||
if !first_attempt {
|
||||
if start.elapsed() >= total_deadline {
|
||||
return Err("KMS failed to become ready within 5 seconds".into());
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…");
|
||||
}
|
||||
Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()),
|
||||
sleep(backoff).await;
|
||||
backoff = (backoff * 2).min(max_backoff);
|
||||
}
|
||||
first_attempt = false;
|
||||
|
||||
let now = tokio::time::Instant::now();
|
||||
if now >= deadline {
|
||||
return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into());
|
||||
}
|
||||
sleep((now + backoff).min(deadline) - now).await;
|
||||
backoff = (backoff * 2).min(max_backoff);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod readiness_tests {
|
||||
use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[tokio::test]
|
||||
async fn kms_readiness_retries_http_success_until_backend_is_healthy() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
|
||||
let address = listener.local_addr().expect("read readiness test server address");
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let server_requests = Arc::clone(&requests);
|
||||
let server = tokio::spawn(async move {
|
||||
for backend_status in ["error", "healthy"] {
|
||||
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
|
||||
let mut request = Vec::new();
|
||||
let mut chunk = [0_u8; 1024];
|
||||
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let read = socket.read(&mut chunk).await.expect("read readiness request");
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
request.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
server_requests.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
let body = format!(r#"{{"backend_status":"{backend_status}"}}"#);
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
socket.write_all(response.as_bytes()).await.expect("write readiness response");
|
||||
match get_kms_status(base_url, access_key, secret_key).await {
|
||||
Ok(status) => {
|
||||
info!("KMS is ready (status: {})", status);
|
||||
return Ok(());
|
||||
}
|
||||
});
|
||||
|
||||
wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key")
|
||||
.await
|
||||
.expect("KMS should become ready after the healthy response");
|
||||
|
||||
let observed_requests = requests.load(Ordering::SeqCst);
|
||||
server.abort();
|
||||
assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kms_readiness_deadline_covers_a_stalled_status_request() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
|
||||
let address = listener.local_addr().expect("read readiness test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let _ = socket.read(&mut request).await.expect("read readiness request");
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)),
|
||||
)
|
||||
.await
|
||||
.expect("readiness helper must enforce its own deadline");
|
||||
|
||||
server.abort();
|
||||
assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline");
|
||||
Err(e) => {
|
||||
if start.elapsed() >= total_deadline {
|
||||
return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
|
||||
}
|
||||
warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
// Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
|
||||
@@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
// Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
|
||||
@@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
let bucket = "copy-object-self-copy-bucket-default-sse-test";
|
||||
|
||||
@@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
let bucket = "copy-object-version-restore-sse-test";
|
||||
|
||||
@@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -147,7 +147,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -24,6 +24,7 @@ use super::common::{
|
||||
test_sse_kms_encryption, test_sse_s3_encryption,
|
||||
};
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::info;
|
||||
|
||||
/// Comprehensive test: Full KMS workflow with all encryption types
|
||||
@@ -34,7 +35,7 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -102,7 +103,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -136,7 +137,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -207,7 +208,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -252,7 +253,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -44,7 +44,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -117,7 +117,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -209,7 +209,7 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box<dyn std::err
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -284,7 +284,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -371,7 +371,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = Arc::new(kms_env.base_env.create_s3_client());
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -478,7 +478,7 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -37,7 +37,7 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -127,7 +127,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -218,7 +218,7 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -401,7 +401,7 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -46,7 +46,7 @@ async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
|
||||
.expect("Failed to start RustFS with Local KMS");
|
||||
|
||||
// Wait a moment for RustFS to fully start up and initialize KMS
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
|
||||
|
||||
@@ -127,7 +127,7 @@ async fn test_local_kms_key_isolation() {
|
||||
.expect("Failed to start RustFS with Local KMS");
|
||||
|
||||
// Wait a moment for RustFS to fully start up and initialize KMS
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
|
||||
|
||||
@@ -227,7 +227,7 @@ async fn test_local_kms_large_file() {
|
||||
.expect("Failed to start RustFS with Local KMS");
|
||||
|
||||
// Wait a moment for RustFS to fully start up and initialize KMS
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
|
||||
|
||||
@@ -309,7 +309,7 @@ async fn test_local_kms_multipart_upload() {
|
||||
.expect("Failed to start RustFS with Local KMS");
|
||||
|
||||
// Wait for KMS initialization
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
//! multipart upload behaviour.
|
||||
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::{error, info};
|
||||
|
||||
use super::common::{
|
||||
@@ -44,8 +45,8 @@ impl VaultKmsTestContext {
|
||||
|
||||
start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
|
||||
|
||||
// Wait for KMS to finish initialising.
|
||||
super::common::wait_for_kms_ready(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
|
||||
// Allow Vault to finish initialising token auth and transit engine.
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
|
||||
Ok(Self { env })
|
||||
}
|
||||
|
||||
@@ -39,6 +39,9 @@ mod kms_edge_cases_test;
|
||||
#[cfg(test)]
|
||||
mod kms_fault_recovery_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_runner;
|
||||
|
||||
#[cfg(test)]
|
||||
mod bucket_default_encryption_test;
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -89,7 +89,7 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -310,7 +310,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -435,7 +435,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -0,0 +1,499 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
#![allow(dead_code)]
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Unified KMS test suite runner
|
||||
//!
|
||||
//! This module provides a unified interface for running KMS tests with categorization,
|
||||
//! filtering, and comprehensive reporting capabilities.
|
||||
|
||||
use crate::common::init_logging;
|
||||
use std::time::Instant;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// Test category for organization and filtering
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum TestCategory {
|
||||
CoreFunctionality,
|
||||
MultipartEncryption,
|
||||
EdgeCases,
|
||||
FaultRecovery,
|
||||
Comprehensive,
|
||||
Performance,
|
||||
}
|
||||
|
||||
impl TestCategory {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
TestCategory::CoreFunctionality => "core-functionality",
|
||||
TestCategory::MultipartEncryption => "multipart-encryption",
|
||||
TestCategory::EdgeCases => "edge-cases",
|
||||
TestCategory::FaultRecovery => "fault-recovery",
|
||||
TestCategory::Comprehensive => "comprehensive",
|
||||
TestCategory::Performance => "performance",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Test definition with metadata
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestDefinition {
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
pub category: TestCategory,
|
||||
pub estimated_duration: Duration,
|
||||
pub is_critical: bool,
|
||||
}
|
||||
|
||||
impl TestDefinition {
|
||||
pub fn new(
|
||||
name: impl Into<String>,
|
||||
description: impl Into<String>,
|
||||
category: TestCategory,
|
||||
estimated_duration: Duration,
|
||||
is_critical: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
description: description.into(),
|
||||
category,
|
||||
estimated_duration,
|
||||
is_critical,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Test execution result
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestResult {
|
||||
pub test_name: String,
|
||||
pub category: TestCategory,
|
||||
pub success: bool,
|
||||
pub duration: Duration,
|
||||
pub error_message: Option<String>,
|
||||
}
|
||||
|
||||
impl TestResult {
|
||||
pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
|
||||
Self {
|
||||
test_name,
|
||||
category,
|
||||
success: true,
|
||||
duration,
|
||||
error_message: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
|
||||
Self {
|
||||
test_name,
|
||||
category,
|
||||
success: false,
|
||||
duration,
|
||||
error_message: Some(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Comprehensive test suite configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestSuiteConfig {
|
||||
pub categories: Vec<TestCategory>,
|
||||
pub include_critical_only: bool,
|
||||
pub max_duration: Option<Duration>,
|
||||
pub parallel_execution: bool,
|
||||
}
|
||||
|
||||
impl Default for TestSuiteConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
categories: vec![
|
||||
TestCategory::CoreFunctionality,
|
||||
TestCategory::MultipartEncryption,
|
||||
TestCategory::EdgeCases,
|
||||
TestCategory::FaultRecovery,
|
||||
TestCategory::Comprehensive,
|
||||
],
|
||||
include_critical_only: false,
|
||||
max_duration: None,
|
||||
parallel_execution: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Unified KMS test suite runner
|
||||
pub struct KMSTestSuite {
|
||||
tests: Vec<TestDefinition>,
|
||||
config: TestSuiteConfig,
|
||||
}
|
||||
|
||||
impl KMSTestSuite {
|
||||
/// Create a new test suite with default configuration
|
||||
pub fn new() -> Self {
|
||||
let tests = vec![
|
||||
// Core Functionality Tests
|
||||
TestDefinition::new(
|
||||
"test_local_kms_end_to_end",
|
||||
"End-to-end KMS test with all encryption types",
|
||||
TestCategory::CoreFunctionality,
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_local_kms_key_isolation",
|
||||
"Test KMS key isolation and security",
|
||||
TestCategory::CoreFunctionality,
|
||||
Duration::from_secs(45),
|
||||
true,
|
||||
),
|
||||
// Multipart Encryption Tests
|
||||
TestDefinition::new(
|
||||
"test_local_kms_multipart_upload",
|
||||
"Test large file multipart upload with encryption",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(120),
|
||||
true,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step1_basic_single_file_encryption",
|
||||
"Basic single file encryption test",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(30),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step2_basic_multipart_upload_without_encryption",
|
||||
"Basic multipart upload without encryption",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(45),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step3_multipart_upload_with_sse_s3",
|
||||
"Multipart upload with SSE-S3 encryption",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step4_large_multipart_upload_with_encryption",
|
||||
"Large file multipart upload with encryption",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(90),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step5_all_encryption_types_multipart",
|
||||
"All encryption types multipart test",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(120),
|
||||
true,
|
||||
),
|
||||
// Edge Cases Tests
|
||||
TestDefinition::new(
|
||||
"test_kms_zero_byte_file_encryption",
|
||||
"Test encryption of zero-byte files",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(20),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_single_byte_file_encryption",
|
||||
"Test encryption of single-byte files",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(20),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_multipart_boundary_conditions",
|
||||
"Test multipart upload boundary conditions",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(45),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_invalid_key_scenarios",
|
||||
"Test invalid key scenarios",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(30),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_concurrent_encryption",
|
||||
"Test concurrent encryption operations",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(60),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_key_validation_security",
|
||||
"Test key validation security",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(30),
|
||||
false,
|
||||
),
|
||||
// Fault Recovery Tests
|
||||
TestDefinition::new(
|
||||
"test_kms_key_directory_unavailable",
|
||||
"Test KMS when key directory is unavailable",
|
||||
TestCategory::FaultRecovery,
|
||||
Duration::from_secs(45),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_corrupted_key_files",
|
||||
"Test KMS with corrupted key files",
|
||||
TestCategory::FaultRecovery,
|
||||
Duration::from_secs(30),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_multipart_upload_interruption",
|
||||
"Test multipart upload interruption recovery",
|
||||
TestCategory::FaultRecovery,
|
||||
Duration::from_secs(60),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_resource_constraints",
|
||||
"Test KMS under resource constraints",
|
||||
TestCategory::FaultRecovery,
|
||||
Duration::from_secs(90),
|
||||
false,
|
||||
),
|
||||
// Comprehensive Tests
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_kms_full_workflow",
|
||||
"Full KMS workflow comprehensive test",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(300),
|
||||
true,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_stress_test",
|
||||
"KMS stress test with large datasets",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(400),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_key_isolation",
|
||||
"Comprehensive key isolation test",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(180),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_concurrent_operations",
|
||||
"Comprehensive concurrent operations test",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(240),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_performance_benchmark",
|
||||
"KMS performance benchmark test",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(360),
|
||||
false,
|
||||
),
|
||||
];
|
||||
|
||||
Self {
|
||||
tests,
|
||||
config: TestSuiteConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Configure the test suite
|
||||
pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
|
||||
self.config = config;
|
||||
self
|
||||
}
|
||||
|
||||
/// Filter tests based on category
|
||||
pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
|
||||
self.tests.iter().filter(|test| &test.category == category).collect()
|
||||
}
|
||||
|
||||
/// Filter tests based on criticality
|
||||
pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
|
||||
self.tests.iter().filter(|test| test.is_critical).collect()
|
||||
}
|
||||
|
||||
/// Get test summary by category
|
||||
pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
|
||||
let mut summary = std::collections::HashMap::new();
|
||||
for test in &self.tests {
|
||||
summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
|
||||
}
|
||||
summary
|
||||
}
|
||||
|
||||
/// Run the complete test suite
|
||||
pub async fn run_test_suite(&self) -> Vec<TestResult> {
|
||||
init_logging();
|
||||
info!("🚀 Starting unified KMS test suite");
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut results = Vec::new();
|
||||
|
||||
// Filter tests based on configuration
|
||||
let tests_to_run: Vec<&TestDefinition> = self
|
||||
.tests
|
||||
.iter()
|
||||
.filter(|test| self.config.categories.contains(&test.category))
|
||||
.filter(|test| !self.config.include_critical_only || test.is_critical)
|
||||
.collect();
|
||||
|
||||
info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
|
||||
for (i, test) in tests_to_run.iter().enumerate() {
|
||||
info!(" {}. {} ({})", i + 1, test.name, test.category.as_str());
|
||||
}
|
||||
|
||||
// Execute tests
|
||||
for (i, test_def) in tests_to_run.iter().enumerate() {
|
||||
info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
|
||||
info!(" 📝 Description: {}", test_def.description);
|
||||
info!(" 🏷️ Category: {}", test_def.category.as_str());
|
||||
info!(" ⏱️ Estimated duration: {:?}", test_def.estimated_duration);
|
||||
|
||||
let test_start = Instant::now();
|
||||
let result = self.run_single_test(test_def).await;
|
||||
let test_duration = test_start.elapsed();
|
||||
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
|
||||
results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
|
||||
}
|
||||
Err(e) => {
|
||||
error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
|
||||
results.push(TestResult::failure(
|
||||
test_def.name.clone(),
|
||||
test_def.category.clone(),
|
||||
test_duration,
|
||||
e.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Add delay between tests to avoid resource conflicts
|
||||
if i < tests_to_run.len() - 1 {
|
||||
debug!("⏸️ Waiting two seconds before the next test...");
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
}
|
||||
|
||||
let total_duration = start_time.elapsed();
|
||||
self.print_test_summary(&results, total_duration);
|
||||
|
||||
results
|
||||
}
|
||||
|
||||
/// Run a single test by dispatching to the appropriate test function
|
||||
async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
// This is a placeholder for test dispatch logic
|
||||
// In a real implementation, this would dispatch to actual test functions
|
||||
warn!("⚠️ Test '{}' is not implemented in the unified runner; skipping", test_def.name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Print comprehensive test summary
|
||||
fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
|
||||
info!("📊 KMS test suite summary");
|
||||
info!("⏱️ Total duration: {:.2} seconds", total_duration.as_secs_f64());
|
||||
info!("📈 Total tests: {}", results.len());
|
||||
|
||||
let passed = results.iter().filter(|r| r.success).count();
|
||||
let failed = results.iter().filter(|r| !r.success).count();
|
||||
|
||||
info!("✅ Passed: {}", passed);
|
||||
info!("❌ Failed: {}", failed);
|
||||
info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
|
||||
|
||||
// Summary by category
|
||||
let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
|
||||
for result in results {
|
||||
let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
|
||||
*total += 1;
|
||||
if result.success {
|
||||
*passed_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
info!("📊 Category summary:");
|
||||
for (category, (total, passed_count)) in category_summary {
|
||||
info!(
|
||||
" 🏷️ {}: {}/{} ({:.1}%)",
|
||||
category.as_str(),
|
||||
passed_count,
|
||||
total,
|
||||
(passed_count as f64 / total as f64) * 100.0
|
||||
);
|
||||
}
|
||||
|
||||
// List failed tests
|
||||
if failed > 0 {
|
||||
warn!("❌ Failing tests:");
|
||||
for result in results.iter().filter(|r| !r.success) {
|
||||
warn!(" - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Quick test suite for critical tests only
|
||||
#[tokio::test]
|
||||
async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let config = TestSuiteConfig {
|
||||
categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
|
||||
include_critical_only: true,
|
||||
max_duration: Some(Duration::from_secs(600)), // 10 minutes max
|
||||
parallel_execution: false,
|
||||
};
|
||||
|
||||
let suite = KMSTestSuite::new().with_config(config);
|
||||
let results = suite.run_test_suite().await;
|
||||
|
||||
let failed_count = results.iter().filter(|r| !r.success).count();
|
||||
if failed_count > 0 {
|
||||
return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
|
||||
}
|
||||
|
||||
info!("✅ All critical tests passed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full comprehensive test suite
|
||||
#[tokio::test]
|
||||
async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let suite = KMSTestSuite::new();
|
||||
let results = suite.run_test_suite().await;
|
||||
|
||||
let total_tests = results.len();
|
||||
let failed_count = results.iter().filter(|r| !r.success).count();
|
||||
let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
|
||||
|
||||
info!("📊 Full suite success rate: {:.1}%", success_rate);
|
||||
|
||||
// Allow up to 10% failure rate for non-critical tests
|
||||
if success_rate < 90.0 {
|
||||
return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
|
||||
}
|
||||
|
||||
info!("✅ Full test suite succeeded");
|
||||
Ok(())
|
||||
}
|
||||
@@ -16,7 +16,6 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
|
||||
use rustfs_signer::pre_sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use s3s::Body;
|
||||
@@ -977,8 +976,7 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
|
||||
.await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "object-lambda-e2e-disabled-target";
|
||||
let key = "input.txt";
|
||||
@@ -994,24 +992,17 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
|
||||
tokio::fs::create_dir_all(&queue_dir).await?;
|
||||
let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
|
||||
let directive = format!(
|
||||
"notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
|
||||
);
|
||||
let disable_response = signed_request(
|
||||
http::Method::PUT,
|
||||
&config_url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
Some(directive.into_bytes()),
|
||||
Some("text/plain"),
|
||||
configure_webhook_target_with_key_values(
|
||||
&env,
|
||||
"transformer",
|
||||
vec![
|
||||
("endpoint", "http://127.0.0.1:9/transform".to_string()),
|
||||
("auth_token", "secret-token".to_string()),
|
||||
("enable", "off".to_string()),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let disable_status = disable_response.status();
|
||||
let disable_body = disable_response.text().await?;
|
||||
assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");
|
||||
wait_for_target_visibility(&env, "transformer").await?;
|
||||
|
||||
let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
|
||||
let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
|
||||
@@ -1030,8 +1021,7 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
|
||||
.await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "object-lambda-e2e-invalid-endpoint";
|
||||
|
||||
@@ -1074,8 +1064,7 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
|
||||
.await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let response = send_configure_webhook_target_request(
|
||||
&env,
|
||||
@@ -1184,8 +1173,6 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
|
||||
init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
|
||||
cluster.set_env(ENV_NOTIFY_ENABLE, "true");
|
||||
cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "listen-notification-cluster";
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::common::{RustFSTestClusterEnvironment, init_logging};
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::CompletedMultipartUpload;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
@@ -42,18 +43,32 @@ async fn list_parts_reports_missing_upload(
|
||||
}
|
||||
}
|
||||
|
||||
async fn multipart_listing_reports_missing_upload(
|
||||
async fn complete_reports_missing_upload(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
upload_id: &str,
|
||||
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
|
||||
|
||||
Ok(!result
|
||||
.uploads()
|
||||
.iter()
|
||||
.any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
|
||||
let result = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().build())
|
||||
.send()
|
||||
.await;
|
||||
match result {
|
||||
Ok(_) => Ok(false),
|
||||
Err(SdkError::ServiceError(err)) => {
|
||||
let code = err.err().meta().code().unwrap_or("");
|
||||
if code == "NoSuchUpload" {
|
||||
Ok(true)
|
||||
} else {
|
||||
Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
|
||||
}
|
||||
}
|
||||
Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_cleanup_on_all_nodes(
|
||||
@@ -66,8 +81,8 @@ async fn wait_for_cleanup_on_all_nodes(
|
||||
let mut all_cleaned = true;
|
||||
for (idx, client) in clients.iter().enumerate() {
|
||||
let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
|
||||
let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
|
||||
if !(list_parts_missing && listing_missing) {
|
||||
let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
|
||||
if !(list_parts_missing && complete_missing) {
|
||||
info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
|
||||
all_cleaned = false;
|
||||
break;
|
||||
@@ -131,10 +146,6 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
|
||||
1,
|
||||
"multipart upload should be visible before background cleanup"
|
||||
);
|
||||
assert!(
|
||||
!multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
|
||||
"multipart upload listing should contain the upload before background cleanup"
|
||||
);
|
||||
|
||||
wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
|
||||
|
||||
|
||||
@@ -317,6 +317,8 @@ pub mod config {
|
||||
}
|
||||
|
||||
pub mod data_usage {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_usage::seed_bucket_usage_memory_for_test;
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
@@ -328,8 +330,6 @@ pub mod data_usage {
|
||||
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
|
||||
store_data_usage_in_backend,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
|
||||
}
|
||||
|
||||
pub mod disk {
|
||||
|
||||
@@ -3425,44 +3425,6 @@ mod tests {
|
||||
assert!(mutexes.contains_key("second"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_all_targets_publishes_disable_proxy_on_target_client() {
|
||||
// The read-proxy selector (replication_proxy::get_proxy_targets) skips
|
||||
// targets whose TargetClient carries disable_proxy — the persisted
|
||||
// per-target opt-out must survive client publication.
|
||||
let sys = BucketTargetSys::default();
|
||||
let target = |arn: &str, disable_proxy: bool| BucketTarget {
|
||||
arn: arn.to_string(),
|
||||
endpoint: "192.168.1.10:9000".to_string(),
|
||||
target_bucket: "target-bucket".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
disable_proxy,
|
||||
credentials: Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: None,
|
||||
expiration: None,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let targets = BucketTargets {
|
||||
targets: vec![target("arn:proxied", false), target("arn:opted-out", true)],
|
||||
};
|
||||
|
||||
sys.update_all_targets("bucket", Some(&targets)).await;
|
||||
|
||||
let proxied = sys
|
||||
.get_remote_target_client("bucket", "arn:proxied")
|
||||
.await
|
||||
.expect("client should be published");
|
||||
assert!(!proxied.disable_proxy);
|
||||
let opted_out = sys
|
||||
.get_remote_target_client("bucket", "arn:opted-out")
|
||||
.await
|
||||
.expect("client should be published");
|
||||
assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn target_updates_serialize_client_build_through_publication_per_bucket() {
|
||||
let sys = Arc::new(BucketTargetSys::default());
|
||||
|
||||
@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
|
||||
.map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
let asz = oi.get_actual_size_or_physical();
|
||||
let asz = oi.get_actual_size().unwrap_or_default();
|
||||
let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
|
||||
let checksum = if ssec { oi.checksum.clone() } else { None };
|
||||
|
||||
|
||||
@@ -1412,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
};
|
||||
let mut replication_state = oi.replication_state();
|
||||
replication_state.replicate_decision_str = dsc.to_string();
|
||||
let actual_size = oi.get_actual_size_or_physical();
|
||||
let actual_size = oi.get_actual_size().unwrap_or_default();
|
||||
|
||||
Ok(ReplicateObjectInfo {
|
||||
name: oi.name.clone(),
|
||||
|
||||
@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
|
||||
.map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
|
||||
version_id: object_info.version_id.map(|version_id| version_id.to_string()),
|
||||
etag: object_info.etag.as_deref(),
|
||||
actual_size: object_info.get_actual_size_or_physical(),
|
||||
actual_size: object_info.get_actual_size().unwrap_or_default(),
|
||||
delete_marker: object_info.delete_marker,
|
||||
content_type: object_info.content_type.as_deref(),
|
||||
content_encoding: object_info.content_encoding.as_deref(),
|
||||
@@ -542,20 +542,6 @@ mod tests {
|
||||
assert!(replication_target_head_is_newer_null_version(&source, &target));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_source_uses_physical_size_for_unknown_compressed_object() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
actual_size: -1,
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(replication_source_object(&source).actual_size, 128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_target_head_content_matches_compare_etag_only() {
|
||||
let source = ObjectInfo {
|
||||
|
||||
@@ -85,6 +85,7 @@ const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const REPLICATION_STATS_MAX_MESSAGE_SIZE: usize = 8 * 1024 * 1024;
|
||||
const BUCKET_METADATA_RELOAD_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Error for a peer that reported `success = false` without an `error_info` payload.
|
||||
///
|
||||
@@ -1328,27 +1329,38 @@ impl PeerRestClient {
|
||||
}
|
||||
|
||||
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
scanner_maintenance_change,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
|
||||
let response = client.load_bucket_metadata(request).await?.into_inner();
|
||||
if !response.success {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
|
||||
}
|
||||
Ok(())
|
||||
let result = tokio::time::timeout(BUCKET_METADATA_RELOAD_TIMEOUT, async {
|
||||
let result = self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
|
||||
if let Err(err) = &result
|
||||
&& Self::is_network_like_error(err)
|
||||
{
|
||||
self.prepare_retry().await;
|
||||
return self.load_bucket_metadata_once(bucket, scanner_maintenance_change).await;
|
||||
}
|
||||
.await,
|
||||
)
|
||||
result
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
|
||||
self.finalize_result(result).await
|
||||
}
|
||||
|
||||
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(LoadBucketMetadataRequest {
|
||||
bucket: bucket.to_string(),
|
||||
scanner_maintenance_change,
|
||||
});
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
request.set_timeout(BUCKET_METADATA_RELOAD_TIMEOUT);
|
||||
|
||||
let response = client.load_bucket_metadata(request).await?.into_inner();
|
||||
if !response.success {
|
||||
if let Some(msg) = response.error_info {
|
||||
return Err(Error::other(msg));
|
||||
}
|
||||
return Err(peer_failure_without_details("load_bucket_metadata", Some(bucket)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn delete_bucket_metadata(&self, bucket: &str) -> Result<()> {
|
||||
|
||||
@@ -42,9 +42,8 @@ use futures::lock::Mutex;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
|
||||
INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
|
||||
INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
|
||||
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
|
||||
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
|
||||
};
|
||||
use rustfs_protos::ChannelClass;
|
||||
use rustfs_protos::evict_failed_connection;
|
||||
@@ -99,7 +98,6 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
|
||||
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
|
||||
const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
|
||||
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
|
||||
const BATCH_METADATA_RPC_OFF: &str = "off";
|
||||
const BATCH_METADATA_RPC_AUTO: &str = "auto";
|
||||
const BATCH_METADATA_RPC_ON: &str = "on";
|
||||
@@ -204,8 +202,7 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
|
||||
}
|
||||
|
||||
fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
|
||||
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
|
||||
.or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
|
||||
rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
|
||||
.or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
|
||||
.as_deref()
|
||||
.map(parse_batch_metadata_rpc_mode)
|
||||
@@ -1829,12 +1826,6 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
|
||||
}
|
||||
}
|
||||
|
||||
fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
|
||||
if let Some(started_at) = started_at {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
|
||||
}
|
||||
}
|
||||
|
||||
/// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
|
||||
/// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
|
||||
/// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
|
||||
@@ -1945,27 +1936,6 @@ fn decode_batch_read_version_response_items(
|
||||
Ok(batch_read_version_resps)
|
||||
}
|
||||
|
||||
fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
|
||||
req.items
|
||||
.iter()
|
||||
.fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
|
||||
total
|
||||
.saturating_add(item.org_volume.len())
|
||||
.saturating_add(item.volume.len())
|
||||
.saturating_add(item.path.len())
|
||||
.saturating_add(item.version_id.len())
|
||||
})
|
||||
}
|
||||
|
||||
fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
|
||||
response
|
||||
.batch_read_version_resps
|
||||
.iter()
|
||||
.map(String::len)
|
||||
.sum::<usize>()
|
||||
.saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
|
||||
}
|
||||
|
||||
fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
file_info.validate_for_metadata_read().map_err(Into::into)
|
||||
}
|
||||
@@ -2867,19 +2837,14 @@ impl DiskAPI for RemoteDisk {
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
|
||||
let batch_read_version_req = compat_json(&req)?;
|
||||
let batch_read_version_req_bin = encode_msgpack(&req)?;
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
|
||||
let request_payload_bytes = batch_read_version_attribution_enabled
|
||||
.then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
|
||||
|
||||
let batch_result = self
|
||||
.execute_with_timeout_for_op(
|
||||
"batch_read_version",
|
||||
move || async move {
|
||||
let disk = self.disk_ref().await;
|
||||
let disk_len = disk.len();
|
||||
let mut client = self
|
||||
.get_bulk_client()
|
||||
.await
|
||||
@@ -2890,20 +2855,9 @@ impl DiskAPI for RemoteDisk {
|
||||
batch_read_version_req_bin: batch_read_version_req_bin.into(),
|
||||
});
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
|
||||
if let Some(request_payload_bytes) = request_payload_bytes {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
|
||||
request_payload_bytes.saturating_add(disk_len),
|
||||
);
|
||||
}
|
||||
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
|
||||
let response = match client.batch_read_version(request).await {
|
||||
Ok(response) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
response.into_inner()
|
||||
}
|
||||
Ok(response) => response.into_inner(),
|
||||
Err(status) if status.code() == Code::Unimplemented => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
if mode.should_fallback_on_unimplemented() {
|
||||
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
|
||||
warn!(
|
||||
@@ -2920,7 +2874,6 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
|
||||
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -2933,33 +2886,14 @@ impl DiskAPI for RemoteDisk {
|
||||
);
|
||||
return Err(Error::from(status));
|
||||
}
|
||||
Err(status) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
|
||||
return Err(Error::from(status));
|
||||
}
|
||||
Err(status) => return Err(Error::from(status)),
|
||||
};
|
||||
|
||||
if !response.success {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
|
||||
batch_read_version_response_payload_len(&response),
|
||||
);
|
||||
let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
|
||||
match decode_batch_read_version_response_items(response, &self.endpoint) {
|
||||
Ok(batch_read_version_resps) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
|
||||
Ok(Some(batch_read_version_resps))
|
||||
}
|
||||
Err(err) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
@@ -4687,7 +4621,6 @@ mod tests {
|
||||
} else {
|
||||
"file version not found".to_string()
|
||||
},
|
||||
error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4807,7 +4740,6 @@ mod tests {
|
||||
fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
|
||||
(ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
|
||||
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
|
||||
],
|
||||
@@ -4817,25 +4749,10 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
|
||||
(ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
|
||||
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
|
||||
],
|
||||
|| {
|
||||
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
|
||||
(ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
|
||||
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
|
||||
],
|
||||
|
||||
@@ -14,10 +14,9 @@
|
||||
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
|
||||
INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
global_internode_metrics,
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -94,59 +93,6 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_stage_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
stage,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics()
|
||||
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
bytes,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
bytes,
|
||||
);
|
||||
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_error() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
|
||||
@@ -248,13 +248,10 @@ impl Config {
|
||||
let shard_size = shard_size as usize;
|
||||
// Keep the historical two-data-shard object budget while preventing
|
||||
// wider EC layouts from multiplying the maximum inline object size.
|
||||
// Use div_ceil to match the shard_file_size calculation (which also uses
|
||||
// div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline
|
||||
// for objects right at the threshold.
|
||||
let inline_block = if self.initialized && self.inline_block_explicit {
|
||||
self.inline_block
|
||||
} else {
|
||||
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
(DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
};
|
||||
|
||||
if versioned {
|
||||
|
||||
+472
-1835
File diff suppressed because it is too large
Load Diff
@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
|
||||
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
|
||||
multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
|
||||
object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
|
||||
object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
|
||||
range::HTTPRangeSpec,
|
||||
};
|
||||
use crate::{
|
||||
@@ -414,66 +414,6 @@ fn apply_delete_objects_results(
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_delete_accounting_results(
|
||||
accounting: &mut [Option<DeleteAccounting>],
|
||||
set_objects: &[DelObj],
|
||||
set_accounting: &[Option<DeleteAccounting>],
|
||||
) {
|
||||
for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
|
||||
accounting[obj.orig_idx] = value.clone();
|
||||
}
|
||||
}
|
||||
|
||||
impl Sets {
|
||||
pub(crate) async fn delete_objects_with_accounting(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
let mut del_errs = vec![None; objects.len()];
|
||||
let mut accounting = vec![None; objects.len()];
|
||||
let mut set_obj_map = HashMap::new();
|
||||
|
||||
for (i, obj) in objects.iter().enumerate() {
|
||||
let idx = self.get_hashed_set_index(obj.object_name.as_str());
|
||||
set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
|
||||
orig_idx: i,
|
||||
obj: obj.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
|
||||
let mut futures = FuturesUnordered::new();
|
||||
let bucket = bucket.to_owned();
|
||||
|
||||
for (set_index, set_objects) in set_obj_map {
|
||||
let disks = self.get_disks(set_index);
|
||||
let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
|
||||
let bucket = bucket.clone();
|
||||
let opts = opts.clone();
|
||||
let semaphore = semaphore.clone();
|
||||
futures.push(async move {
|
||||
let _permit = semaphore
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("delete_objects semaphore should remain open");
|
||||
let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
|
||||
(set_objects, deleted, errors, accounting)
|
||||
});
|
||||
}
|
||||
|
||||
while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
|
||||
apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
|
||||
apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
|
||||
}
|
||||
|
||||
(del_objects, del_errs, accounting)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for Sets {
|
||||
type Error = Error;
|
||||
@@ -715,8 +655,65 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
|
||||
let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
|
||||
(deleted, errors)
|
||||
// Default return value
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
|
||||
let mut del_errs = Vec::with_capacity(objects.len());
|
||||
for _ in 0..objects.len() {
|
||||
del_errs.push(None)
|
||||
}
|
||||
|
||||
let mut set_obj_map = HashMap::new();
|
||||
|
||||
// hash key
|
||||
for (i, obj) in objects.iter().enumerate() {
|
||||
let idx = self.get_hashed_set_index(obj.object_name.as_str());
|
||||
|
||||
if !set_obj_map.contains_key(&idx) {
|
||||
set_obj_map.insert(
|
||||
idx,
|
||||
vec![DelObj {
|
||||
// set_idx: idx,
|
||||
orig_idx: i,
|
||||
obj: obj.clone(),
|
||||
}],
|
||||
);
|
||||
} else if let Some(val) = set_obj_map.get_mut(&idx) {
|
||||
val.push(DelObj {
|
||||
// set_idx: idx,
|
||||
orig_idx: i,
|
||||
obj: obj.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
|
||||
let mut futures = FuturesUnordered::new();
|
||||
let bucket = bucket.to_string();
|
||||
|
||||
for (k, v) in set_obj_map {
|
||||
let disks = self.get_disks(k);
|
||||
let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
|
||||
let bucket = bucket.clone();
|
||||
let opts = opts.clone();
|
||||
let semaphore = semaphore.clone();
|
||||
|
||||
futures.push(async move {
|
||||
let _permit = semaphore
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("delete_objects semaphore should remain open");
|
||||
let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
|
||||
(v, dobjects, errs)
|
||||
});
|
||||
}
|
||||
|
||||
while let Some((v, dobjects, errs)) = futures.next().await {
|
||||
apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
|
||||
}
|
||||
|
||||
(del_objects, del_errs)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
|
||||
@@ -26,7 +26,7 @@ use crate::storage_api_contracts::{
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectOperations as _},
|
||||
};
|
||||
use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
|
||||
use crate::store::ECStore;
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
|
||||
@@ -856,6 +856,7 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
|
||||
fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
|
||||
is_unversioned_data_movement_object(source)
|
||||
&& is_unversioned_data_movement_object(target)
|
||||
&& !target.delete_marker
|
||||
&& source
|
||||
.mod_time
|
||||
.zip(target.mod_time)
|
||||
@@ -1027,7 +1028,6 @@ pub(crate) enum SourceCleanupError {
|
||||
pub(crate) struct SourceCleanupBucketFence<'a> {
|
||||
pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
|
||||
pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
|
||||
}
|
||||
|
||||
fn ensure_source_cleanup_versions_match(
|
||||
@@ -1065,9 +1065,7 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
|
||||
struct SourceCleanupDeleteBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
fence_pending: tokio::sync::Notify,
|
||||
arrived: tokio::sync::Notify,
|
||||
is_paused: AtomicBool,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
@@ -1081,7 +1079,7 @@ pub(crate) struct SourceCleanupDeleteBarrier {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<SourceCleanupDeleteBarrierState>>>> =
|
||||
static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1094,22 +1092,15 @@ impl SourceCleanupDeleteBarrier {
|
||||
let state = Arc::new(SourceCleanupDeleteBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
fence_pending: tokio::sync::Notify::new(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
is_paused: AtomicBool::new(false),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison");
|
||||
assert!(
|
||||
!barriers
|
||||
.iter()
|
||||
.any(|barrier| barrier.bucket == bucket && barrier.object == object),
|
||||
"source cleanup delete barrier must be unique per object"
|
||||
);
|
||||
barriers.push(Arc::clone(&state));
|
||||
assert!(slot.is_none(), "source cleanup delete barrier must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
@@ -1119,58 +1110,35 @@ impl SourceCleanupDeleteBarrier {
|
||||
.expect("source cleanup should reach the pre-delete barrier");
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_fence_pending(&self) {
|
||||
tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified())
|
||||
.await
|
||||
.expect("source cleanup should attempt the fixed mutation fence");
|
||||
}
|
||||
|
||||
pub(crate) fn is_paused(&self) -> bool {
|
||||
self.state.is_paused.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) {
|
||||
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison")
|
||||
.iter()
|
||||
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.fence_pending.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for SourceCleanupDeleteBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison");
|
||||
barriers.retain(|state| !Arc::ptr_eq(state, &self.state));
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
|
||||
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
let barrier = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison")
|
||||
.iter()
|
||||
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.is_paused.store(true, Ordering::Release);
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
@@ -1186,20 +1154,11 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
op_label: &str,
|
||||
) -> std::result::Result<ObjectInfo, SourceCleanupError> {
|
||||
let cleanup_key = encode_dir_object(object);
|
||||
let source_guard = if bucket_fence
|
||||
.object_mutation_fence
|
||||
.is_some_and(SourceCleanupMutationFence::source_lock_covered)
|
||||
{
|
||||
None
|
||||
} else {
|
||||
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
|
||||
Some(
|
||||
ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(Error::from)?,
|
||||
)
|
||||
};
|
||||
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
|
||||
let _guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(Error::from)?;
|
||||
|
||||
if bucket_fence
|
||||
.lifecycle_guard
|
||||
@@ -1209,14 +1168,6 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
"{op_label}: bucket incarnation fence was lost before source cleanup"
|
||||
))));
|
||||
}
|
||||
if bucket_fence
|
||||
.object_mutation_fence
|
||||
.is_some_and(SourceCleanupMutationFence::is_lock_lost)
|
||||
{
|
||||
return Err(SourceCleanupError::Storage(Error::other(format!(
|
||||
"{op_label}: object mutation fence was lost before source cleanup"
|
||||
))));
|
||||
}
|
||||
|
||||
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
|
||||
|
||||
@@ -1231,12 +1182,7 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(source_guard) = source_guard.as_ref() {
|
||||
opts.add_namespace_lock_guard(source_guard);
|
||||
}
|
||||
if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
|
||||
object_mutation_fence.add_namespace_lock_fence(&mut opts);
|
||||
}
|
||||
opts.add_namespace_lock_guard(&_guard);
|
||||
if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
|
||||
opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
|
||||
}
|
||||
@@ -1384,37 +1330,6 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn migrate_decommission_object(
|
||||
store: Arc<ECStore>,
|
||||
pool_idx: usize,
|
||||
bucket: String,
|
||||
rd: GetObjectReader,
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
) -> Result<()> {
|
||||
let source = rd.object_info.clone();
|
||||
let _mutation_fence = store
|
||||
.acquire_decommission_object_mutation_fence(&bucket, &source.name)
|
||||
.await?;
|
||||
let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
|
||||
.await?
|
||||
.ok_or(Error::FileNotFound)?;
|
||||
if !is_equivalent_data_movement_object_identity(&source, ¤t, true, false) {
|
||||
return Err(Error::FileNotFound);
|
||||
}
|
||||
|
||||
migrate_object_inner(
|
||||
store,
|
||||
pool_idx,
|
||||
bucket,
|
||||
rd,
|
||||
source_bucket_incarnation_id,
|
||||
op_label,
|
||||
Some(&_mutation_fence),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn migrate_object(
|
||||
store: Arc<ECStore>,
|
||||
pool_idx: usize,
|
||||
@@ -1422,18 +1337,6 @@ pub(crate) async fn migrate_object(
|
||||
rd: GetObjectReader,
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
) -> Result<()> {
|
||||
migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
|
||||
}
|
||||
|
||||
async fn migrate_object_inner(
|
||||
store: Arc<ECStore>,
|
||||
pool_idx: usize,
|
||||
bucket: String,
|
||||
rd: GetObjectReader,
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<()> {
|
||||
let object_info = rd.object_info.clone();
|
||||
let has_part_checksums = object_info
|
||||
@@ -1447,7 +1350,7 @@ async fn migrate_object_inner(
|
||||
let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
|
||||
new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
|
||||
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
|
||||
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts)
|
||||
.await
|
||||
{
|
||||
Ok(res) => res,
|
||||
@@ -1545,7 +1448,7 @@ async fn migrate_object_inner(
|
||||
if let Err(err) = store
|
||||
.clone()
|
||||
.complete_multipart_upload_for_data_movement(
|
||||
(target_pool_idx, mutation_fence),
|
||||
target_pool_idx,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&res.upload_id,
|
||||
@@ -1706,7 +1609,7 @@ async fn migrate_object_inner(
|
||||
let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
|
||||
put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
let (target_pool_idx, put_result) = store
|
||||
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
|
||||
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts)
|
||||
.await
|
||||
.map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
|
||||
if let Err(err) = put_result {
|
||||
@@ -3638,47 +3541,25 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_accepts_only_newer_null_delete_marker() {
|
||||
for version_id in [None, Some(Uuid::nil())] {
|
||||
let source = ObjectInfo {
|
||||
version_id,
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
delete_marker: true,
|
||||
etag: None,
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
fn test_precondition_conflict_rejects_newer_delete_marker() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
delete_marker: true,
|
||||
etag: None,
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
assert!(
|
||||
resolve_data_movement_overwrite_resume_result(
|
||||
&Error::PreconditionFailed,
|
||||
Ok(Some(target.clone())),
|
||||
&source,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
.expect("newer null delete marker should be evaluated")
|
||||
);
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("delete marker conflict should be evaluated");
|
||||
|
||||
let mut same_time = target.clone();
|
||||
same_time.mod_time = source.mod_time;
|
||||
assert!(
|
||||
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,)
|
||||
.expect("same-generation null delete marker should be rejected")
|
||||
);
|
||||
|
||||
let mut versioned = target;
|
||||
versioned.version_id = Some(Uuid::new_v4());
|
||||
assert!(
|
||||
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,)
|
||||
.expect("a UUID delete marker must not erase a null source version")
|
||||
);
|
||||
}
|
||||
assert!(!should_resume);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1391,37 +1391,7 @@ impl BucketUsageAccumulator {
|
||||
}
|
||||
|
||||
pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
|
||||
// A compressed object may carry -1 while the transformed size is unknown
|
||||
// (legacy streaming sentinel). In that case the persisted physical size
|
||||
// is still a valid accounting floor; every other negative value is corrupt.
|
||||
// An explicit negative `actual-size` metadata value is corrupt, however:
|
||||
// the sentinel is only valid in the in-memory/object-part field written by
|
||||
// the legacy streaming path, not as a persisted declared size.
|
||||
let compressed = object.is_compressed();
|
||||
if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
if object
|
||||
.parts
|
||||
.iter()
|
||||
.any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
|
||||
{
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
|
||||
.filter(|value| !value.is_empty());
|
||||
if declared_actual_size
|
||||
.as_deref()
|
||||
.and_then(|value| value.parse::<i64>().ok())
|
||||
.is_some_and(|size| size < 0)
|
||||
{
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
let logical_size = match object.get_actual_size().map_err(Error::other)? {
|
||||
size if size == -1 && compressed && declared_actual_size.is_none() => None,
|
||||
size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
|
||||
_ => return Err(Error::PartMissingOrCorrupt),
|
||||
};
|
||||
let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
let persisted_part_size = if object.parts.is_empty() {
|
||||
u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
} else {
|
||||
@@ -1429,8 +1399,12 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
|
||||
// Compressed streaming objects persist -1 when the transformed
|
||||
// part size is unknown. The physical part size remains a valid
|
||||
// quota floor; reject only non-negative values that overflow.
|
||||
let actual_size = if part.actual_size == -1 {
|
||||
0
|
||||
let actual_size = if part.actual_size < 0 {
|
||||
if object.is_compressed() {
|
||||
0
|
||||
} else {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
} else {
|
||||
u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
};
|
||||
@@ -1438,7 +1412,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
|
||||
total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
|
||||
})?
|
||||
};
|
||||
Ok(logical_size.unwrap_or(0).max(persisted_part_size))
|
||||
Ok(logical_size.max(persisted_part_size))
|
||||
}
|
||||
|
||||
type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
@@ -3346,80 +3320,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
let object = ObjectInfo {
|
||||
size: 400,
|
||||
actual_size: -1,
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_object_size_rejects_compressed_part_sum_overflow() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
let object = ObjectInfo {
|
||||
size: 1,
|
||||
user_defined: Arc::new(metadata),
|
||||
parts: Arc::new(vec![
|
||||
rustfs_filemeta::ObjectPartInfo {
|
||||
actual_size: i64::MAX,
|
||||
..Default::default()
|
||||
},
|
||||
rustfs_filemeta::ObjectPartInfo {
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
},
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
let corrupt_object = ObjectInfo {
|
||||
size: 400,
|
||||
actual_size: -2,
|
||||
user_defined: Arc::new(metadata.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
|
||||
|
||||
let corrupt_part = ObjectInfo {
|
||||
size: 400,
|
||||
user_defined: Arc::new(metadata),
|
||||
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
size: 400,
|
||||
actual_size: -2,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
|
||||
|
||||
@@ -858,7 +858,6 @@ const EVENT_DISK_LOCAL_DIRECT_IO_FALLBACK: &str = "disk_local_direct_io_fallback
|
||||
#[cfg(target_os = "linux")]
|
||||
const EVENT_DISK_LOCAL_URING_LATCH_OFF: &str = "disk_local_uring_latch_off";
|
||||
const EVENT_DISK_LOCAL_DELETE_FAILED: &str = "disk_local_delete_failed";
|
||||
const EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED: &str = "disk_local_delete_rollback_failed";
|
||||
const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
|
||||
const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
|
||||
const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
|
||||
@@ -6107,43 +6106,6 @@ impl LocalDisk {
|
||||
Ok((bytes, modtime))
|
||||
}
|
||||
|
||||
async fn write_missing_delete_marker(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
object_dir: &Path,
|
||||
xl_path: &Path,
|
||||
rollback_dir: Option<Uuid>,
|
||||
) -> Result<()> {
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
let rollback_path = object_dir.join(rollback_dir.to_string());
|
||||
fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
|
||||
fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
if let Err(err) = self.write_metadata("", volume, path, fi).await {
|
||||
if let Some(rollback_dir) = rollback_dir
|
||||
&& let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
result = "failed",
|
||||
volume,
|
||||
path,
|
||||
rollback_dir = %rollback_dir,
|
||||
error = ?restore_err,
|
||||
"Disk local delete rollback failed"
|
||||
);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
|
||||
@@ -6161,20 +6123,7 @@ impl LocalDisk {
|
||||
return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
|
||||
}
|
||||
|
||||
let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
|
||||
Ok(data) => data,
|
||||
Err(DiskError::FileNotFound) => {
|
||||
// `deleted` alone can be an explicit marker purge; only
|
||||
// `mark_deleted` may create metadata that was not present.
|
||||
let Some(delete_marker) = fis.iter().find(|fi| fi.deleted && fi.mark_deleted).cloned() else {
|
||||
return Err(DiskError::FileNotFound);
|
||||
};
|
||||
return self
|
||||
.write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir)
|
||||
.await;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?;
|
||||
|
||||
if data.is_empty() {
|
||||
return Err(DiskError::FileNotFound);
|
||||
@@ -10473,9 +10422,29 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
if fi.deleted && force_del_marker {
|
||||
return self
|
||||
.write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir)
|
||||
.await;
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
let rollback_path = file_path.join(rollback_dir.to_string());
|
||||
fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
|
||||
fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
if let Err(err) = self.write_metadata("", volume, path, fi).await {
|
||||
if let Some(rollback_dir) = rollback_dir
|
||||
&& let Err(restore_err) =
|
||||
restore_delete_rollback(file_path.as_path(), &xl_path, rollback_dir, &self.publication_root).await
|
||||
{
|
||||
warn!(
|
||||
volume,
|
||||
path,
|
||||
rollback_dir = %rollback_dir,
|
||||
error = ?restore_err,
|
||||
"failed to restore metadata after delete marker commit error"
|
||||
);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
return if fi.version_id.is_some() {
|
||||
|
||||
@@ -44,8 +44,6 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_DISK: &str = "disk";
|
||||
const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
|
||||
const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
|
||||
const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;
|
||||
|
||||
pub fn part_transaction_path(part_path: &str) -> String {
|
||||
match part_path.rsplit_once('/') {
|
||||
@@ -64,7 +62,6 @@ use bytes::Bytes;
|
||||
use endpoint::Endpoint;
|
||||
use error::DiskError;
|
||||
use error::{Error, Result};
|
||||
use futures::stream::{self, StreamExt};
|
||||
use local::LocalDisk;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_madmin::info_commands::DiskMetrics;
|
||||
@@ -420,14 +417,6 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
|
||||
match self {
|
||||
@@ -1039,47 +1028,36 @@ where
|
||||
D: DiskAPI + ?Sized,
|
||||
{
|
||||
validate_batch_read_version_item_count(req.items.len())?;
|
||||
let parallelism = batch_read_version_server_parallelism();
|
||||
|
||||
let mut responses = stream::iter(req.items.into_iter().enumerate())
|
||||
.map(|(index, item)| async move {
|
||||
match disk
|
||||
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
|
||||
.await
|
||||
{
|
||||
Ok(file_info) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path,
|
||||
version_id: item.version_id,
|
||||
success: true,
|
||||
file_info,
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
},
|
||||
Err(err) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path,
|
||||
version_id: item.version_id,
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: err.to_string(),
|
||||
error_code: err.to_u32(),
|
||||
},
|
||||
}
|
||||
})
|
||||
.buffer_unordered(parallelism)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
responses.sort_unstable_by_key(|response| response.index);
|
||||
let mut responses = Vec::with_capacity(req.items.len());
|
||||
for (index, item) in req.items.iter().enumerate() {
|
||||
let response = match disk
|
||||
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
|
||||
.await
|
||||
{
|
||||
Ok(file_info) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path.clone(),
|
||||
version_id: item.version_id.clone(),
|
||||
success: true,
|
||||
file_info,
|
||||
error: String::new(),
|
||||
},
|
||||
Err(err) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path.clone(),
|
||||
version_id: item.version_id.clone(),
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: err.to_string(),
|
||||
},
|
||||
};
|
||||
responses.push(response);
|
||||
}
|
||||
|
||||
Ok(responses)
|
||||
}
|
||||
|
||||
fn batch_read_version_server_parallelism() -> usize {
|
||||
rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
|
||||
.clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct CheckPartsResp {
|
||||
pub results: Vec<usize>,
|
||||
@@ -1344,8 +1322,6 @@ pub struct BatchReadVersionResp {
|
||||
pub success: bool,
|
||||
pub file_info: FileInfo,
|
||||
pub error: String,
|
||||
#[serde(default)]
|
||||
pub error_code: u32,
|
||||
}
|
||||
|
||||
pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
|
||||
@@ -1441,26 +1417,6 @@ mod tests {
|
||||
assert!(!partial_valid_location.valid());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
|
||||
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
|
||||
assert_eq!(batch_read_version_server_parallelism(), 4);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_server_parallelism_honors_env_with_bounds() {
|
||||
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
|
||||
assert_eq!(batch_read_version_server_parallelism(), 8);
|
||||
});
|
||||
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
|
||||
assert_eq!(batch_read_version_server_parallelism(), 1);
|
||||
});
|
||||
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
|
||||
assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
|
||||
});
|
||||
}
|
||||
|
||||
/// Test FileInfoVersions find_version_index
|
||||
#[test]
|
||||
fn test_file_info_versions_find_version_index() {
|
||||
|
||||
@@ -81,14 +81,6 @@ pub fn shutdown_background_monitors() {
|
||||
cluster::rpc::shutdown_background_monitors();
|
||||
}
|
||||
|
||||
/// Publish that the process is ready to serve user-object GET traffic.
|
||||
///
|
||||
/// Experimental metadata coalescing is allowed to run only after this point so
|
||||
/// startup and internal metadata reads keep the original per-disk path.
|
||||
pub fn mark_get_metadata_read_version_coalescing_service_ready() {
|
||||
runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod rio_tests {
|
||||
#[test]
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::storage_api_contracts::{
|
||||
pub struct NamespaceLockFence {
|
||||
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
|
||||
#[cfg(test)]
|
||||
forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
|
||||
forced_lost: Arc<std::sync::atomic::AtomicBool>,
|
||||
}
|
||||
|
||||
impl Debug for NamespaceLockFence {
|
||||
@@ -40,17 +40,13 @@ impl NamespaceLockFence {
|
||||
Self {
|
||||
signals: Arc::default(),
|
||||
#[cfg(test)]
|
||||
forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
|
||||
forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
#[cfg(test)]
|
||||
if self
|
||||
.forced_lost
|
||||
.iter()
|
||||
.any(|lost| lost.load(std::sync::atomic::Ordering::Acquire))
|
||||
{
|
||||
if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return true;
|
||||
}
|
||||
self.signals.iter().any(|signal| signal.is_lost())
|
||||
@@ -61,26 +57,27 @@ impl NamespaceLockFence {
|
||||
}
|
||||
|
||||
fn extend(&mut self, other: &Self) {
|
||||
if !Arc::ptr_eq(&self.signals, &other.signals) {
|
||||
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
|
||||
if Arc::ptr_eq(&self.signals, &other.signals) {
|
||||
return;
|
||||
}
|
||||
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
|
||||
#[cfg(test)]
|
||||
if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
|
||||
Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
|
||||
if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
|
||||
self.forced_lost.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn lost_for_test() -> Self {
|
||||
let fence = Self::new();
|
||||
fence.forced_lost[0].store(true, std::sync::atomic::Ordering::Release);
|
||||
fence.forced_lost.store(true, std::sync::atomic::Ordering::Release);
|
||||
fence
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn loss_handle_for_test() -> (Self, Arc<std::sync::atomic::AtomicBool>) {
|
||||
let fence = Self::new();
|
||||
(fence.clone(), Arc::clone(&fence.forced_lost[0]))
|
||||
(fence.clone(), Arc::clone(&fence.forced_lost))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,13 +411,6 @@ impl ObjectOptions {
|
||||
self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
|
||||
self.namespace_lock_fence
|
||||
.get_or_insert_with(NamespaceLockFence::new)
|
||||
.extend(fence);
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
|
||||
self.lifecycle_delete_all_journal
|
||||
.get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
|
||||
@@ -699,9 +689,6 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
pub fn get_actual_size(&self) -> std::io::Result<i64> {
|
||||
if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
|
||||
return Err(std::io::Error::other("invalid negative actual size"));
|
||||
}
|
||||
if self.actual_size > 0 {
|
||||
return Ok(self.actual_size);
|
||||
}
|
||||
@@ -713,25 +700,10 @@ impl ObjectInfo {
|
||||
let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
return Ok(size);
|
||||
}
|
||||
if self.actual_size == -1 && self.parts.is_empty() {
|
||||
return Ok(-1);
|
||||
}
|
||||
let mut actual_size = 0_i64;
|
||||
let mut unknown = false;
|
||||
for part in self.parts.iter() {
|
||||
match part.actual_size {
|
||||
-1 => unknown = true,
|
||||
size if size >= 0 => {
|
||||
actual_size = actual_size
|
||||
.checked_add(size)
|
||||
.ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
|
||||
}
|
||||
_ => return Err(std::io::Error::other("invalid negative compressed part size")),
|
||||
}
|
||||
}
|
||||
if unknown {
|
||||
return Ok(-1);
|
||||
}
|
||||
let mut actual_size = 0;
|
||||
self.parts.iter().for_each(|part| {
|
||||
actual_size += part.actual_size;
|
||||
});
|
||||
if actual_size == 0 && actual_size != self.size {
|
||||
return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
|
||||
}
|
||||
@@ -746,18 +718,6 @@ impl ObjectInfo {
|
||||
Ok(self.size)
|
||||
}
|
||||
|
||||
/// Returns a non-negative size for client and replication boundaries.
|
||||
///
|
||||
/// Compressed legacy metadata can retain the internal `-1` unknown-size
|
||||
/// sentinel. Those boundaries cannot emit a negative length, so they use
|
||||
/// the persisted physical size while quota accounting keeps the sentinel
|
||||
/// distinction in [`crate::data_usage::quota_object_size`].
|
||||
pub fn get_actual_size_or_physical(&self) -> i64 {
|
||||
self.get_actual_size()
|
||||
.map(|size| if size >= 0 { size } else { self.size.max(0) })
|
||||
.unwrap_or_else(|_| self.size.max(0))
|
||||
}
|
||||
|
||||
pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
|
||||
let mut version_id = fi.version_id;
|
||||
|
||||
|
||||
@@ -25,10 +25,7 @@ use lazy_static::lazy_static;
|
||||
use rustfs_lock::client::LockClient;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
sync::{Arc, OnceLock},
|
||||
time::SystemTime,
|
||||
};
|
||||
use tokio::sync::{OnceCell, RwLock};
|
||||
@@ -40,16 +37,6 @@ pub const DISK_MIN_INODES: u64 = 1000;
|
||||
pub const DISK_FILL_FRACTION: f64 = 0.99;
|
||||
pub const DISK_RESERVE_FRACTION: f64 = 0.15;
|
||||
|
||||
static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
|
||||
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
|
||||
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
// Global singletons for backward compatibility with MinIO port.
|
||||
// These should be migrated to AppContext over time.
|
||||
// See issue #730 for migration plan.
|
||||
|
||||
@@ -160,10 +160,6 @@ pub struct InstanceContext {
|
||||
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
|
||||
/// Replaces the process-global cancel-token static.
|
||||
background_cancel_token: OnceLock<CancellationToken>,
|
||||
/// Serializes decommission data-movement operations with cancellation and
|
||||
/// a subsequent restart. Readers are held across one object side effect;
|
||||
/// the transition path takes the writer after cancelling the routine.
|
||||
decommission_operation_gate: Arc<RwLock<()>>,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
@@ -204,7 +200,6 @@ impl InstanceContext {
|
||||
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
|
||||
bucket_metadata_sys: std::sync::Mutex::new(None),
|
||||
background_cancel_token: OnceLock::new(),
|
||||
decommission_operation_gate: Arc::new(RwLock::new(())),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
@@ -223,10 +218,6 @@ impl InstanceContext {
|
||||
self.lock_manager.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
|
||||
Arc::clone(&self.decommission_operation_gate)
|
||||
}
|
||||
|
||||
/// Install the application-owned object-encryption resolver once.
|
||||
pub fn set_object_encryption_resolver(
|
||||
&self,
|
||||
|
||||
@@ -256,10 +256,6 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
|
||||
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
|
||||
cycle_max_objects: metrics.cycle_max_objects,
|
||||
cycle_max_directories: metrics.cycle_max_directories,
|
||||
cycle_timeout_total: metrics.cycle_timeout_total,
|
||||
cycle_recovery_required_total: metrics.cycle_recovery_required_total,
|
||||
cycle_last_progress_age: metrics.cycle_last_progress_age,
|
||||
leader_lease_without_progress: metrics.leader_lease_without_progress,
|
||||
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
|
||||
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
|
||||
scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
|
||||
@@ -615,10 +611,6 @@ mod test {
|
||||
current_started: chrono_to_jiff_timestamp(current_started),
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
cycle_timeout_total: 3,
|
||||
cycle_recovery_required_total: 2,
|
||||
cycle_last_progress_age: 17,
|
||||
leader_lease_without_progress: true,
|
||||
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
|
||||
source: "usage".to_string(),
|
||||
cycles: 2,
|
||||
@@ -630,10 +622,6 @@ mod test {
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
assert_eq!(scanner.cycle_timeout_total, 3);
|
||||
assert_eq!(scanner.cycle_recovery_required_total, 2);
|
||||
assert_eq!(scanner.cycle_last_progress_age, 17);
|
||||
assert!(scanner.leader_lease_without_progress);
|
||||
let usage = scanner
|
||||
.partial_cycles_by_source
|
||||
.iter()
|
||||
|
||||
@@ -334,7 +334,6 @@ impl ECStore {
|
||||
lifecycle_guard: bucket_incarnation_fence
|
||||
.as_ref()
|
||||
.and_then(|guard| guard.namespace_lock_guard()),
|
||||
..Default::default()
|
||||
},
|
||||
"rebalance",
|
||||
),
|
||||
|
||||
@@ -53,12 +53,11 @@ use crate::diagnostics::get::{
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout};
|
||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
||||
use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
|
||||
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
||||
STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
|
||||
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
|
||||
PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
@@ -76,7 +75,7 @@ use std::{
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
sync::{
|
||||
Arc, OnceLock,
|
||||
OnceLock,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
task::{Context, Poll},
|
||||
@@ -95,242 +94,6 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String {
|
||||
[bucket, object].join("/")
|
||||
}
|
||||
|
||||
fn read_version_coalescing_enabled() -> bool {
|
||||
let enabled = || {
|
||||
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
|
||||
.is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on"))
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
{
|
||||
enabled()
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
static ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
*ENABLED.get_or_init(enabled)
|
||||
}
|
||||
}
|
||||
|
||||
fn read_version_coalescing_delay() -> Duration {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let micros = rustfs_utils::get_env_u64(
|
||||
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
|
||||
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
|
||||
);
|
||||
Duration::from_micros(micros)
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
static DELAY: OnceLock<Duration> = OnceLock::new();
|
||||
*DELAY.get_or_init(|| {
|
||||
Duration::from_micros(rustfs_utils::get_env_u64(
|
||||
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
|
||||
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct CoalescedReadVersionRequest {
|
||||
item: BatchReadVersionItem,
|
||||
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
struct ReadVersionCoalescerKey {
|
||||
disk: usize,
|
||||
incl_free_versions: bool,
|
||||
read_data: bool,
|
||||
healing: bool,
|
||||
}
|
||||
|
||||
impl ReadVersionCoalescerKey {
|
||||
fn new(disk: &DiskStore, opts: &ReadOptions) -> Self {
|
||||
Self {
|
||||
disk: Arc::as_ptr(disk) as usize,
|
||||
incl_free_versions: opts.incl_free_versions,
|
||||
read_data: opts.read_data,
|
||||
healing: opts.healing,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReadVersionCoalescer {
|
||||
lanes: HashMap<ReadVersionCoalescerKey, Vec<CoalescedReadVersionRequest>>,
|
||||
}
|
||||
|
||||
fn read_version_coalescer() -> &'static Mutex<ReadVersionCoalescer> {
|
||||
static COALESCER: OnceLock<Mutex<ReadVersionCoalescer>> = OnceLock::new();
|
||||
COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default()))
|
||||
}
|
||||
|
||||
fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
|
||||
counter!(
|
||||
METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL,
|
||||
"event" => event,
|
||||
"item_count" => item_count.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
async fn read_version_via_coalescer(
|
||||
disk: DiskStore,
|
||||
org_bucket: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
if !allow_coalescing || !read_version_coalescing_enabled() {
|
||||
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
|
||||
}
|
||||
if !matches!(disk.as_ref(), Disk::Remote(_)) {
|
||||
record_read_version_coalescer_event("bypass_non_remote", 1);
|
||||
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
|
||||
}
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let item = BatchReadVersionItem {
|
||||
org_volume: org_bucket.to_string(),
|
||||
volume: bucket.to_string(),
|
||||
path: object.to_string(),
|
||||
version_id: version_id.to_string(),
|
||||
};
|
||||
let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
|
||||
let pending = {
|
||||
let mut coalescer = read_version_coalescer().lock().await;
|
||||
let lane = coalescer.lanes.entry(lane_key).or_default();
|
||||
let schedule_delayed_flush = lane.is_empty();
|
||||
lane.push(CoalescedReadVersionRequest { item, tx });
|
||||
if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
|
||||
coalescer.lanes.remove(&lane_key)
|
||||
} else if schedule_delayed_flush {
|
||||
let disk = disk.clone();
|
||||
let task_opts = *opts;
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(read_version_coalescing_delay()).await;
|
||||
flush_read_version_coalescer_lane(lane_key, disk, task_opts).await;
|
||||
});
|
||||
None
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(pending) = pending {
|
||||
flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
|
||||
}
|
||||
|
||||
rx.await
|
||||
.unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
|
||||
}
|
||||
|
||||
async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
|
||||
let pending = {
|
||||
let mut coalescer = read_version_coalescer().lock().await;
|
||||
coalescer.lanes.remove(&lane_key).unwrap_or_default()
|
||||
};
|
||||
flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await;
|
||||
}
|
||||
|
||||
async fn flush_read_version_coalescer_pending(
|
||||
lane_key: ReadVersionCoalescerKey,
|
||||
disk: DiskStore,
|
||||
opts: ReadOptions,
|
||||
pending: Vec<CoalescedReadVersionRequest>,
|
||||
) {
|
||||
if pending.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
{
|
||||
let mut observed_paths = HashSet::new();
|
||||
for request in &pending {
|
||||
if observed_paths.insert(request.item.path.as_str()) {
|
||||
disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut senders = Vec::with_capacity(pending.len());
|
||||
let mut items = Vec::with_capacity(pending.len());
|
||||
for request in pending {
|
||||
senders.push(request.tx);
|
||||
items.push(request.item);
|
||||
}
|
||||
|
||||
let expected_items = items.clone();
|
||||
record_read_version_coalescer_event("attempted_batch", items.len());
|
||||
let result =
|
||||
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
};
|
||||
match result {
|
||||
Ok(responses) => {
|
||||
let results = map_batch_read_version_responses(&expected_items, responses);
|
||||
for (tx, result) in senders.into_iter().zip(results) {
|
||||
let _ = tx.send(result);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let message = err.to_string();
|
||||
for tx in senders {
|
||||
let _ = tx.send(Err(DiskError::other(message.clone())));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_batch_read_version_responses(
|
||||
expected_items: &[BatchReadVersionItem],
|
||||
responses: Vec<BatchReadVersionResp>,
|
||||
) -> Vec<crate::disk::error::Result<FileInfo>> {
|
||||
let mut results = (0..expected_items.len())
|
||||
.map(|_| Err(DiskError::other("coalesced read_version response missing")))
|
||||
.collect::<Vec<_>>();
|
||||
let mut seen = vec![false; expected_items.len()];
|
||||
for response in responses {
|
||||
let Some(expected) = expected_items.get(response.index) else {
|
||||
continue;
|
||||
};
|
||||
let Some(slot) = results.get_mut(response.index) else {
|
||||
continue;
|
||||
};
|
||||
if seen[response.index] {
|
||||
*slot = Err(DiskError::other("coalesced read_version response duplicate index"));
|
||||
continue;
|
||||
}
|
||||
seen[response.index] = true;
|
||||
if response.path != expected.path || response.version_id != expected.version_id {
|
||||
*slot = Err(DiskError::other("coalesced read_version response identity mismatch"));
|
||||
} else {
|
||||
*slot = if response.success {
|
||||
Ok(response.file_info)
|
||||
} else {
|
||||
Err(batch_read_version_response_error(response.error_code, response.error))
|
||||
};
|
||||
}
|
||||
}
|
||||
results
|
||||
}
|
||||
|
||||
fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError {
|
||||
match DiskError::from_u32(error_code) {
|
||||
Some(DiskError::Io(_)) | None => DiskError::other(error),
|
||||
Some(error) => error,
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn bounded_metadata_fanout_order(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -370,15 +133,11 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
|
||||
order
|
||||
}
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::sync::{Mutex, RwLock, oneshot};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
|
||||
pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
|
||||
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
|
||||
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
|
||||
const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
|
||||
const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
|
||||
pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
|
||||
/// Default reader-setup strategy for the GET read path (rustfs/backlog#1215,
|
||||
/// #1159, #923).
|
||||
@@ -2597,7 +2356,6 @@ impl SetDisks {
|
||||
false,
|
||||
true,
|
||||
0,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
Ok((ress, errors))
|
||||
@@ -2628,36 +2386,6 @@ impl SetDisks {
|
||||
true,
|
||||
caller_allows_early_stop,
|
||||
default_parity_count,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object(
|
||||
disks: &[Option<DiskStore>],
|
||||
org_bucket: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
read_data: bool,
|
||||
incl_free_versions: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
default_parity_count: usize,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
|
||||
Self::read_all_fileinfo_inner(
|
||||
disks,
|
||||
org_bucket,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
read_data,
|
||||
false,
|
||||
incl_free_versions,
|
||||
true,
|
||||
caller_allows_early_stop,
|
||||
default_parity_count,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2680,7 +2408,6 @@ impl SetDisks {
|
||||
// subset would fail write quorum (backlog#872 regression).
|
||||
caller_allows_early_stop: bool,
|
||||
default_parity_count: usize,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
|
||||
let early_stop_enabled =
|
||||
caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled());
|
||||
@@ -2697,7 +2424,6 @@ impl SetDisks {
|
||||
healing,
|
||||
incl_free_versions,
|
||||
default_parity_count,
|
||||
allow_coalescing,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -2720,7 +2446,6 @@ impl SetDisks {
|
||||
healing,
|
||||
incl_free_versions,
|
||||
observe,
|
||||
allow_coalescing,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2736,7 +2461,6 @@ impl SetDisks {
|
||||
healing: bool,
|
||||
incl_free_versions: bool,
|
||||
observe: bool,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
|
||||
let fanout_start = observe.then(Instant::now);
|
||||
let mut ress = Vec::with_capacity(disks.len());
|
||||
@@ -2768,7 +2492,7 @@ impl SetDisks {
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
@@ -2835,7 +2559,6 @@ impl SetDisks {
|
||||
healing: bool,
|
||||
incl_free_versions: bool,
|
||||
default_parity_count: usize,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
|
||||
let fanout_start = Instant::now();
|
||||
let mut ress = vec![FileInfo::default(); disks.len()];
|
||||
@@ -2884,7 +2607,7 @@ impl SetDisks {
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
@@ -6014,7 +5737,6 @@ pub(crate) mod disk_call_counters {
|
||||
|
||||
/// Kind label for the per-disk `read_version` metadata RPC.
|
||||
pub const KIND_READ_VERSION: &str = "read_version";
|
||||
pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version";
|
||||
|
||||
/// Registry key: (object, kind, disk_index).
|
||||
type CountKey = (String, String, usize);
|
||||
@@ -6738,286 +6460,6 @@ mod tests {
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn metadata_read_version_coalescer_bypasses_local_disks() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "coalesced-read-version-local-bypass-bucket";
|
||||
let object_a = "coalesced-local-object-a";
|
||||
let object_b = "coalesced-local-object-b";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")),
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(object_a);
|
||||
let disks_a = disks.clone();
|
||||
let disks_b = disks.clone();
|
||||
let read_a = tokio::spawn(async move {
|
||||
SetDisks::read_all_fileinfo_observed_for_get_object(
|
||||
&disks_a, "", bucket, object_a, "", false, false, false, 2,
|
||||
)
|
||||
.await
|
||||
.map(|(file_infos, errors, _)| (file_infos, errors))
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
let read_b = tokio::spawn(async move {
|
||||
SetDisks::read_all_fileinfo_observed_for_get_object(
|
||||
&disks_b, "", bucket, object_b, "", false, false, false, 2,
|
||||
)
|
||||
.await
|
||||
.map(|(file_infos, errors, _)| (file_infos, errors))
|
||||
});
|
||||
|
||||
let (metadata_a, errs_a) = read_a
|
||||
.await
|
||||
.expect("first read task should not panic")
|
||||
.expect("first coalesced read should resolve");
|
||||
let (metadata_b, errs_b) = read_b
|
||||
.await
|
||||
.expect("second read task should not panic")
|
||||
.expect("second coalesced read should resolve");
|
||||
|
||||
assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS);
|
||||
assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS);
|
||||
assert!(errs_a.iter().all(Option::is_none));
|
||||
assert!(errs_b.iter().all(Option::is_none));
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
DISKS as u64,
|
||||
"local disks still execute the ordinary per-disk read_version path"
|
||||
);
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
|
||||
0,
|
||||
"GET coalescing targets internode RPC count only and must not batch local disk reads"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_read_version_coalescer_requires_get_object_intent() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "coalesced-read-version-default-bypass-bucket";
|
||||
let object = "default-bypass-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async {
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false)
|
||||
.await
|
||||
.expect("default metadata read should resolve");
|
||||
|
||||
assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64);
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
|
||||
0,
|
||||
"non-GET metadata paths must bypass coalescer even when the env gate is enabled"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_response_mapping_preserves_index_and_errors() {
|
||||
let expected_items = vec![
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
},
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-b".to_string(),
|
||||
version_id: "v-b".to_string(),
|
||||
},
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-c".to_string(),
|
||||
version_id: "v-c".to_string(),
|
||||
},
|
||||
];
|
||||
let ok_file_info = FileInfo {
|
||||
name: "object-a".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let responses = vec![
|
||||
BatchReadVersionResp {
|
||||
index: 2,
|
||||
path: "object-c".to_string(),
|
||||
version_id: "v-c".to_string(),
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: "disk read failed".to_string(),
|
||||
error_code: 0,
|
||||
},
|
||||
BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: true,
|
||||
file_info: ok_file_info,
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
},
|
||||
];
|
||||
|
||||
let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
|
||||
let first = results
|
||||
.next()
|
||||
.expect("slot 0 should exist")
|
||||
.expect("slot 0 should map the success response by index");
|
||||
assert_eq!(first.name, "object-a");
|
||||
|
||||
let missing = results
|
||||
.next()
|
||||
.expect("slot 1 should exist")
|
||||
.expect_err("slot 1 should stay missing");
|
||||
assert!(
|
||||
missing.to_string().contains("response missing"),
|
||||
"unexpected missing response error: {missing}"
|
||||
);
|
||||
|
||||
let failed = results
|
||||
.next()
|
||||
.expect("slot 2 should exist")
|
||||
.expect_err("slot 2 should map the response error");
|
||||
assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}");
|
||||
assert!(results.next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_response_mapping_preserves_typed_not_found_errors() {
|
||||
let expected_items = vec![
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
},
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-b".to_string(),
|
||||
version_id: "v-b".to_string(),
|
||||
},
|
||||
];
|
||||
let results = map_batch_read_version_responses(
|
||||
&expected_items,
|
||||
vec![
|
||||
BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: DiskError::FileNotFound.to_string(),
|
||||
error_code: DiskError::FileNotFound.to_u32(),
|
||||
},
|
||||
BatchReadVersionResp {
|
||||
index: 1,
|
||||
path: "object-b".to_string(),
|
||||
version_id: "v-b".to_string(),
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: DiskError::FileVersionNotFound.to_string(),
|
||||
error_code: DiskError::FileVersionNotFound.to_u32(),
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound)));
|
||||
assert!(matches!(
|
||||
results.get(1).expect("slot 1 should exist"),
|
||||
Err(DiskError::FileVersionNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() {
|
||||
let expected_items = vec![BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
}];
|
||||
let mismatched = map_batch_read_version_responses(
|
||||
&expected_items,
|
||||
vec![BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-b".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: true,
|
||||
file_info: FileInfo {
|
||||
name: "object-b".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
}],
|
||||
)
|
||||
.pop()
|
||||
.expect("slot 0 should exist")
|
||||
.expect_err("identity mismatch should fail closed");
|
||||
assert!(
|
||||
mismatched.to_string().contains("identity mismatch"),
|
||||
"unexpected mismatch error: {mismatched}"
|
||||
);
|
||||
|
||||
let duplicate = map_batch_read_version_responses(
|
||||
&expected_items,
|
||||
vec![
|
||||
BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: true,
|
||||
file_info: FileInfo {
|
||||
name: "object-a".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
},
|
||||
BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: true,
|
||||
file_info: FileInfo {
|
||||
name: "object-a".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
},
|
||||
],
|
||||
)
|
||||
.pop()
|
||||
.expect("slot 0 should exist")
|
||||
.expect_err("duplicate response index should fail closed");
|
||||
assert!(
|
||||
duplicate.to_string().contains("duplicate index"),
|
||||
"unexpected duplicate error: {duplicate}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Isolation guard: unobserved objects record nothing (so parallel tests do
|
||||
/// not inflate one another), and a scope clears its own counts on drop.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -97,7 +97,7 @@ use crate::storage_api_contracts::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
|
||||
},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
|
||||
object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
|
||||
range::HTTPRangeSpec,
|
||||
};
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
@@ -735,12 +735,8 @@ pub(crate) use core::io_primitives::disk_call_counters;
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
@@ -3029,16 +3025,6 @@ pub struct SetDisks {
|
||||
storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
|
||||
}
|
||||
|
||||
// DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones
|
||||
// each endpoint's canonical Arc, so an exact Arc set identifies the lock domain.
|
||||
pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
|
||||
left.iter()
|
||||
.all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client)))
|
||||
&& right
|
||||
.iter()
|
||||
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
|
||||
}
|
||||
|
||||
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
@@ -3614,15 +3600,6 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
/// Whether both sets' namespace-lock implementations cover the same object key.
|
||||
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
|
||||
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
|
||||
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
|
||||
(true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The lock manager this set actually uses (test-only; Phase 5 Slice 3).
|
||||
#[cfg(test)]
|
||||
pub(crate) fn local_lock_manager_for_test(&self) -> &Arc<rustfs_lock::GlobalLockManager> {
|
||||
@@ -4607,11 +4584,11 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool {
|
||||
}
|
||||
|
||||
fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool {
|
||||
opts.delete_marker || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.data_movement)
|
||||
opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement)
|
||||
}
|
||||
|
||||
fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
|
||||
let mut mark_delete = goi.version_id.is_some() || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none());
|
||||
let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none());
|
||||
let mut delete_marker = opts.versioned;
|
||||
|
||||
if opts.version_id.is_some() {
|
||||
|
||||
@@ -32,8 +32,6 @@ use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
use futures::{StreamExt, stream};
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::AtomicBool;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
@@ -67,7 +65,6 @@ impl StaleMultipartCleanupGuard {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MultipartCommitPause {
|
||||
NewUploadBeforeLockLost,
|
||||
PutPartBeforeLockAcquire,
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterRename,
|
||||
@@ -159,72 +156,6 @@ impl Drop for MultipartCommitBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct NewMultipartUploadCommitObservationState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
committed: AtomicBool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct NewMultipartUploadCommitObservation {
|
||||
state: Arc<NewMultipartUploadCommitObservationState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<NewMultipartUploadCommitObservationState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl NewMultipartUploadCommitObservation {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(NewMultipartUploadCommitObservationState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
committed: AtomicBool::new(false),
|
||||
});
|
||||
let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("new multipart upload commit observation mutex should not poison");
|
||||
assert!(slot.is_none(), "new multipart upload commit observation must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) fn committed(&self) -> bool {
|
||||
self.state.committed.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for NewMultipartUploadCommitObservation {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("new multipart upload commit observation mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn observe_new_multipart_upload_commit(bucket: &str, object: &str) {
|
||||
let state = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("new multipart upload commit observation mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|state| state.bucket == bucket && state.object == object)
|
||||
.cloned();
|
||||
if let Some(state) = state {
|
||||
state.committed.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
|
||||
let barrier = {
|
||||
@@ -1684,30 +1615,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement);
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::NewUploadBeforeLockLost).await;
|
||||
if _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "new_multipart_upload_commit",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "new_multipart_upload_outer_lock",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
Self::write_unique_file_info(
|
||||
&shuffle_disks,
|
||||
@@ -1719,8 +1626,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
)
|
||||
.await
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
|
||||
#[cfg(test)]
|
||||
observe_new_multipart_upload_commit(bucket, object);
|
||||
|
||||
// evalDisks
|
||||
|
||||
|
||||
@@ -45,7 +45,6 @@ use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
|
||||
replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
|
||||
};
|
||||
use crate::data_usage::quota_object_size;
|
||||
use crate::diagnostics::get::GetObjectFailureReason;
|
||||
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
|
||||
use crate::error::is_err_invalid_upload_id;
|
||||
@@ -1294,7 +1293,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
(prepared.snapshot, prepared.object_info)
|
||||
} else {
|
||||
match self
|
||||
.get_object_fileinfo_for_get_object_reader(
|
||||
.get_object_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
@@ -2123,27 +2122,14 @@ impl SetDisks {
|
||||
let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
|
||||
|
||||
let put_object_size = known_put_object_storage_size(data.size());
|
||||
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
|
||||
let is_inline_buffer =
|
||||
storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
|
||||
storage_class_config.should_inline(erasure.shard_file_size(put_object_size), erasure.data_shards, opts.versioned);
|
||||
|
||||
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
|
||||
let shard_file_size = shard_file_size_raw;
|
||||
let shard_file_size = erasure.shard_file_size(put_object_size);
|
||||
let shard_size = erasure.shard_size();
|
||||
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
|
||||
let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
|
||||
{
|
||||
use std::io::Write;
|
||||
let msg = format!(
|
||||
"INLINE_DEBUG: bucket={} obj={} size={} shard_fs={} ds={} bs={} inline={} direct={} path={} iblock={} ver={}\n",
|
||||
bucket, object, put_object_size, shard_file_size_raw, erasure.data_shards, fi.erasure.block_size,
|
||||
is_inline_buffer, direct_inline_commit, write_path.metric_label(), storage_class_config.inline_block(), opts.versioned
|
||||
);
|
||||
if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rustfs_inline_debug.log") {
|
||||
let _ = f.write_all(msg.as_bytes());
|
||||
}
|
||||
let _ = std::io::stderr().write_all(msg.as_bytes());
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
|
||||
let writer_setup_stage_start = collect_stage_timing.then(Instant::now);
|
||||
let (mut writers, errors) = if direct_inline_commit {
|
||||
@@ -2497,7 +2483,6 @@ impl SetDisks {
|
||||
})
|
||||
.await?,
|
||||
);
|
||||
notify_put_object_commit_namespace_acquired(bucket, object);
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
{
|
||||
@@ -4645,7 +4630,6 @@ struct PutObjectCommitBarrierState {
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
namespace_pending: tokio::sync::Notify,
|
||||
namespace_acquired: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
@@ -4667,7 +4651,6 @@ impl PutObjectCommitBarrier {
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
namespace_pending: tokio::sync::Notify::new(),
|
||||
namespace_acquired: std::sync::atomic::AtomicBool::new(false),
|
||||
});
|
||||
let mut slot = PUT_OBJECT_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
@@ -4702,10 +4685,6 @@ impl PutObjectCommitBarrier {
|
||||
.await
|
||||
.expect("put object should wait for the namespace lock after leaving the commit barrier");
|
||||
}
|
||||
|
||||
pub fn namespace_acquired(&self) -> bool {
|
||||
self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
@@ -4762,22 +4741,6 @@ fn notify_put_object_commit_namespace_pending(bucket: &str, object: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
|
||||
let barrier = PUT_OBJECT_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("put object commit barrier mutex should not poison")
|
||||
.iter()
|
||||
.find(|barrier| {
|
||||
barrier.bucket == bucket && barrier.object == object && barrier.pause == PutObjectCommitPause::BeforeNamespace
|
||||
})
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DeleteObjectCommitBarrierState {
|
||||
bucket: String,
|
||||
@@ -4787,7 +4750,7 @@ struct DeleteObjectCommitBarrierState {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct DeleteObjectCommitBarrier {
|
||||
struct DeleteObjectCommitBarrier {
|
||||
state: Arc<DeleteObjectCommitBarrierState>,
|
||||
}
|
||||
|
||||
@@ -4797,7 +4760,7 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
|
||||
|
||||
#[cfg(test)]
|
||||
impl DeleteObjectCommitBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(DeleteObjectCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
@@ -4813,13 +4776,13 @@ impl DeleteObjectCommitBarrier {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("delete object should reach the deterministic commit barrier");
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
@@ -5692,18 +5655,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
|
||||
let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
|
||||
(deleted, errors)
|
||||
}
|
||||
|
||||
async fn delete_objects_with_accounting(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
let mut accounting = vec![None; objects.len()];
|
||||
let delete_config_snapshot = opts
|
||||
.delete_replication_config_snapshot
|
||||
.clone()
|
||||
@@ -5793,7 +5745,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
*item = Some(Error::other(message.clone()));
|
||||
}
|
||||
}
|
||||
return (del_objects, del_errs, accounting);
|
||||
return (del_objects, del_errs);
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -5840,22 +5792,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let source_missing = gerr
|
||||
.as_ref()
|
||||
.is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
|
||||
// Resolve accounting from the generation selected under this
|
||||
// object's write lock. A request-layer pre-stat is only an
|
||||
// optimization and cannot identify a concurrent overwrite.
|
||||
let (accounting_size, accounting_version_id, removed_current_object) = if source_missing
|
||||
|| dobj.synthetic_version_id
|
||||
|| set_disk_delete_creates_delete_marker(&check_opts)
|
||||
|| goi.delete_marker
|
||||
{
|
||||
(None, None, false)
|
||||
} else {
|
||||
(
|
||||
quota_object_size(&goi).ok(),
|
||||
goi.version_id.filter(|version_id| !version_id.is_nil()),
|
||||
(dobj.version_id.is_none() || is_explicit_null_version(dobj.version_id)) && !dobj.synthetic_version_id,
|
||||
)
|
||||
};
|
||||
// Normalize both sides before comparing. `goi.version_id` is the
|
||||
// client-facing identity, where `from_file_info` synthesizes
|
||||
// `Some(Uuid::nil())` for a null version on a versioned or
|
||||
@@ -5941,7 +5877,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
if dobj.version_id.is_none() && (version_suspended || versioned) {
|
||||
vr.mod_time = Some(OffsetDateTime::now_utc());
|
||||
vr.deleted = true;
|
||||
vr.mark_deleted = true;
|
||||
if versioned {
|
||||
vr.version_id = Some(Uuid::new_v4());
|
||||
}
|
||||
@@ -5985,12 +5920,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
},
|
||||
replication_state: vr.replication_state_internal.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
accounting[i] = Some(DeleteAccounting {
|
||||
size: accounting_size,
|
||||
version_id: accounting_version_id,
|
||||
removed_current_object,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Only add to vers_map if we hold the lock
|
||||
@@ -6036,7 +5966,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
});
|
||||
}
|
||||
}
|
||||
return (del_objects, del_errs, accounting);
|
||||
return (del_objects, del_errs);
|
||||
}
|
||||
|
||||
let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len());
|
||||
@@ -6274,16 +6204,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
// An accounting identity is actionable only when the delete result is
|
||||
// successful. Never let a failed commit (including a partial quorum
|
||||
// failure) reach the request-layer fast delta path.
|
||||
for (index, err) in del_errs.iter().enumerate() {
|
||||
if err.is_some() {
|
||||
accounting[index] = None;
|
||||
}
|
||||
}
|
||||
|
||||
(del_objects, del_errs, accounting)
|
||||
(del_objects, del_errs)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
@@ -6612,12 +6533,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
obj_info.size = goi.size;
|
||||
// Keep the committed source metadata on the internal delete result so
|
||||
// the request layer can derive canonical accounting for this exact
|
||||
// generation. Delete responses do not expose these fields.
|
||||
obj_info.actual_size = goi.actual_size;
|
||||
obj_info.user_defined = Arc::clone(&goi.user_defined);
|
||||
obj_info.parts = Arc::clone(&goi.parts);
|
||||
obj_info.user_tags = Arc::clone(&goi.user_tags);
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
Ok(obj_info)
|
||||
@@ -7909,113 +7824,6 @@ mod replication_quota_safety_tests {
|
||||
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_returns_canonical_compressed_accounting_size() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "compressed-delete-accounting";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
|
||||
let mut reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
|
||||
.expect("compressed fixture reader should be valid"),
|
||||
);
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
"object",
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
user_defined,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("compressed object should be written");
|
||||
|
||||
let (deleted, errors, accounting) = set_disks
|
||||
.delete_objects_with_accounting(
|
||||
bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: "object".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(errors[0].is_none(), "compressed delete should succeed: {:?}", errors[0]);
|
||||
assert!(deleted[0].found, "the committed object must be reported as found");
|
||||
assert_eq!(accounting[0].as_ref().and_then(|value| value.size), Some(1000));
|
||||
assert!(accounting[0].as_ref().is_some_and(|value| value.version_id.is_none()));
|
||||
assert!(accounting[0].as_ref().is_some_and(|value| value.removed_current_object));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn suspended_delete_marker_does_not_return_body_accounting() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "suspended-delete-accounting";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
|
||||
let mut reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
|
||||
.expect("compressed fixture reader should be valid"),
|
||||
);
|
||||
let suspended_opts = ObjectOptions {
|
||||
version_suspended: true,
|
||||
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
s3s::dto::VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
))),
|
||||
user_defined,
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
};
|
||||
set_disks
|
||||
.put_object(bucket, "object", &mut reader, &suspended_opts)
|
||||
.await
|
||||
.expect("compressed object should be written");
|
||||
|
||||
let (deleted, errors, accounting) = set_disks
|
||||
.delete_objects_with_accounting(
|
||||
bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: "object".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
suspended_opts,
|
||||
)
|
||||
.await;
|
||||
assert!(errors[0].is_none(), "suspended delete should create a marker: {:?}", errors[0]);
|
||||
assert!(deleted[0].delete_marker);
|
||||
assert!(accounting[0].is_none(), "a delete marker must not carry body accounting");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_put_cannot_persist_a_tiny_logical_size() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -11828,7 +11636,6 @@ mod transition_upload_integrity_tests {
|
||||
crate::data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: None,
|
||||
lifecycle_guard: Some(&bucket_guard),
|
||||
..Default::default()
|
||||
},
|
||||
"test_data_movement",
|
||||
)
|
||||
|
||||
@@ -259,33 +259,10 @@ impl SetDisks {
|
||||
read_data: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false)
|
||||
self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
pub(super) async fn get_object_fileinfo_for_get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket)
|
||||
&& crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
|
||||
self.get_object_fileinfo_gated_inner(
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
read_data,
|
||||
caller_allows_early_stop,
|
||||
allow_read_version_coalescing,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
|
||||
/// quorum fanout. Read-before-write callers (object tagging) must use this:
|
||||
/// the returned online-disk set is the write target, and the early-stop
|
||||
@@ -298,20 +275,6 @@ impl SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn get_object_fileinfo_gated_inner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
allow_read_version_coalescing: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
let vid = opts.version_id.clone().unwrap_or_default();
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
@@ -374,34 +337,19 @@ impl SetDisks {
|
||||
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
|
||||
// core/io_primitives.rs); unsafe requests and callers that opt out
|
||||
// (allow_early_stop=false) fall back to full-wait.
|
||||
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing {
|
||||
Self::read_all_fileinfo_observed_for_get_object(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
Self::read_all_fileinfo_observed(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
false,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?
|
||||
};
|
||||
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
false,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?;
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
|
||||
@@ -62,8 +62,8 @@ pub(crate) mod object {
|
||||
use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions,
|
||||
ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
|
||||
DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, ObjectOperations,
|
||||
ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
|
||||
};
|
||||
|
||||
pub(crate) trait EcstoreObjectIO:
|
||||
|
||||
@@ -297,16 +297,10 @@ impl ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
|
||||
use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk};
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
|
||||
use crate::disk::{DiskOption, format::FormatV3, new_disk};
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::store::init_format::{load_format_erasure, save_format_file};
|
||||
use crate::store::init_local_disks_with_instance_ctx;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
|
||||
let format = FormatV3::new(1, 1);
|
||||
@@ -353,51 +347,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
|
||||
let mut pool_endpoints = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("multi-pool heal test disk should be created");
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
|
||||
.expect("test endpoint should parse");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
pool_endpoints.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("heal-owner-pool-{pool_index}"),
|
||||
platform: "test".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(pool_endpoints);
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
.await
|
||||
.expect("multi-pool local disks should initialize");
|
||||
let shutdown = CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx(
|
||||
"127.0.0.1:0".parse().expect("test address should parse"),
|
||||
endpoint_pools,
|
||||
shutdown.clone(),
|
||||
instance_ctx,
|
||||
)
|
||||
.await
|
||||
.expect("multi-pool test store should initialize");
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(temp_dir, store, shutdown)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_pool_scope_selects_only_requested_pool() {
|
||||
let store = minimal_heal_store().await;
|
||||
@@ -557,229 +506,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn unscoped_heal_object_suspended_owner_semantics() {
|
||||
let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
|
||||
let active_object = "active-owner";
|
||||
let suspended_only_object = "suspended-only";
|
||||
let duplicate_object = "duplicate-owner";
|
||||
let marker_object = "marker-owner";
|
||||
let quorum_object = "quorum-owner";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created in all pools");
|
||||
|
||||
let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("active owner object should be written");
|
||||
let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
|
||||
missing_active_disk
|
||||
.delete(
|
||||
&bucket,
|
||||
active_object,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("active owner shard should be removed for repair");
|
||||
assert!(
|
||||
missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
|
||||
"the active owner fixture must start with one missing metadata copy"
|
||||
);
|
||||
|
||||
let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
|
||||
store.pools[1]
|
||||
.put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("suspended owner object should be written");
|
||||
for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
|
||||
let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
|
||||
store.pools[pool_index]
|
||||
.put_object(
|
||||
&bucket,
|
||||
duplicate_object,
|
||||
&mut duplicate_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("duplicate owner object should be written");
|
||||
}
|
||||
let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
|
||||
.clone()
|
||||
.expect("duplicate active owner disk should be online");
|
||||
duplicate_missing_disk
|
||||
.delete(
|
||||
&bucket,
|
||||
duplicate_object,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("duplicate active owner shard should be removed for repair");
|
||||
let history_version = Uuid::new_v4();
|
||||
let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
marker_object,
|
||||
&mut history_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(history_version.to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned marker history should be written");
|
||||
store.pools[0]
|
||||
.delete_object(
|
||||
&bucket,
|
||||
marker_object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete marker should be written");
|
||||
let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("quorum boundary object should be written");
|
||||
{
|
||||
let mut pool_meta = store.pool_meta.write().await;
|
||||
let mut next = PoolMeta::new(&store.pools, &pool_meta);
|
||||
next.pools[1].decommission = Some(PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
});
|
||||
*pool_meta = next;
|
||||
}
|
||||
|
||||
let (_, duplicate_owner) = store
|
||||
.get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("duplicate owner should resolve");
|
||||
assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
|
||||
let (_, active_duplicate_owner) = store
|
||||
.get_latest_object_info_with_idx(
|
||||
&bucket,
|
||||
duplicate_object,
|
||||
&ObjectOptions {
|
||||
skip_decommissioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("active duplicate owner should resolve");
|
||||
assert_eq!(
|
||||
active_duplicate_owner, 0,
|
||||
"suspended duplicate must be excluded from active owner selection"
|
||||
);
|
||||
let (duplicate_result, duplicate_err) = store
|
||||
.handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("duplicate owner heal should complete through the production path");
|
||||
assert_eq!(duplicate_result.object, duplicate_object);
|
||||
assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
|
||||
assert!(
|
||||
duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
|
||||
"production heal must repair the active duplicate owner rather than the suspended owner"
|
||||
);
|
||||
let (marker_info, marker_owner) = store
|
||||
.get_latest_object_info_with_idx(
|
||||
&bucket,
|
||||
marker_object,
|
||||
&ObjectOptions {
|
||||
skip_decommissioned: true,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("latest delete marker should resolve");
|
||||
assert_eq!(marker_owner, 0);
|
||||
assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
|
||||
|
||||
let (active_result, active_err) = store
|
||||
.handle_heal_object(&bucket, active_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("unscoped active-owner heal should complete");
|
||||
assert_eq!(active_result.object, active_object);
|
||||
assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
|
||||
assert!(
|
||||
missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
|
||||
"active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
|
||||
);
|
||||
assert!(
|
||||
store.pools[1]
|
||||
.get_object_info(&bucket, active_object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"the suspended pool must not be written for an active-owner object"
|
||||
);
|
||||
|
||||
let (suspended_result, suspended_err) = store
|
||||
.handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("unscoped suspended-only heal should return a terminal result");
|
||||
assert!(suspended_result.object.is_empty());
|
||||
assert!(matches!(suspended_err, Some(Error::FileNotFound)));
|
||||
assert!(
|
||||
store.pools[1]
|
||||
.get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok(),
|
||||
"suspended-only data must remain untouched when unscoped heal reports absent"
|
||||
);
|
||||
|
||||
let (_, explicit_err) = store
|
||||
.handle_heal_object(
|
||||
&bucket,
|
||||
suspended_only_object,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("explicit suspended-owner heal should return a mapped error");
|
||||
assert!(matches!(explicit_err, Some(Error::SlowDown)));
|
||||
|
||||
let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
let surviving_quorum_disk = original_quorum_disks[3].clone();
|
||||
*store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
|
||||
let (_, quorum_err) = store
|
||||
.handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("quorum boundary heal should return a mapped result");
|
||||
*store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
|
||||
assert!(
|
||||
matches!(quorum_err, Some(Error::ErasureReadQuorum)),
|
||||
"quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handle_heal_format_continues_after_a_pool_error() {
|
||||
let canonical_format = FormatV3::new(1, 3);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -33,7 +33,7 @@ use crate::bucket::utils::check_put_object_part_args;
|
||||
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
|
||||
use crate::cluster::rpc::{RemoteClient, S3PeerSys};
|
||||
use crate::config::storageclass;
|
||||
use crate::core::pools::{DecommissionCanceler, PoolMeta};
|
||||
use crate::core::pools::PoolMeta;
|
||||
use crate::disk::endpoint::{Endpoint, EndpointType};
|
||||
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -151,7 +151,7 @@ pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence};
|
||||
pub(crate) use object::ObjectLockDiagGuard;
|
||||
pub use object::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
SnapshotConsistencyError,
|
||||
@@ -176,7 +176,7 @@ pub struct ECStore {
|
||||
// pub local_disks: Vec<DiskStore>,
|
||||
pub pool_meta: RwLock<PoolMeta>,
|
||||
pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
|
||||
pub decommission_cancelers: RwLock<Vec<Option<DecommissionCanceler>>>,
|
||||
pub decommission_cancelers: RwLock<Vec<Option<CancellationToken>>>,
|
||||
/// Serializes rebalance/decommission start transitions.
|
||||
///
|
||||
/// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`,
|
||||
|
||||
@@ -400,7 +400,7 @@ impl ECStore {
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<MultipartUploadResult> {
|
||||
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts, None)
|
||||
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
|
||||
.await
|
||||
.map(|(res, _, _)| res)
|
||||
}
|
||||
@@ -410,22 +410,20 @@ impl ECStore {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
|
||||
check_new_multipart_args(bucket, object)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
|
||||
if self.single_pool() {
|
||||
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
return self.pools[0]
|
||||
.new_multipart_upload(bucket, object, &opts)
|
||||
.new_multipart_upload(bucket, object, opts)
|
||||
.await
|
||||
.map(|res| (res, 0, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
if opts.data_movement && opts.version_id.is_some() {
|
||||
let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?;
|
||||
let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?;
|
||||
if idx == opts.src_pool_idx {
|
||||
return Err(StorageError::DataMovementOverwriteErr(
|
||||
bucket.to_owned(),
|
||||
@@ -433,9 +431,7 @@ impl ECStore {
|
||||
opts.version_id.clone().unwrap_or_default(),
|
||||
));
|
||||
}
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
||||
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
@@ -458,9 +454,7 @@ impl ECStore {
|
||||
.await?;
|
||||
|
||||
if !res.uploads.is_empty() {
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
||||
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
}
|
||||
@@ -473,9 +467,7 @@ impl ECStore {
|
||||
));
|
||||
}
|
||||
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
||||
Ok((res, idx, opts.expected_bucket_incarnation_id))
|
||||
}
|
||||
|
||||
@@ -712,14 +704,13 @@ impl ECStore {
|
||||
|
||||
pub(crate) async fn complete_multipart_upload_for_data_movement(
|
||||
self: Arc<Self>,
|
||||
target: (usize, Option<&ObjectLockDiagGuard>),
|
||||
target_pool_idx: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
uploaded_parts: Vec<CompletePart>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let (target_pool_idx, mutation_fence) = target;
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||
@@ -748,8 +739,6 @@ impl ECStore {
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
opts.object_lock_config_snapshot = Some(snapshot);
|
||||
}
|
||||
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
#[cfg(test)]
|
||||
pause_data_movement_multipart_before_selected_completion(bucket).await;
|
||||
let pool = self
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -859,7 +859,6 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
|
||||
use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
@@ -1424,14 +1423,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
version_id: Some(version_id),
|
||||
etag,
|
||||
..object_info_with_mod_time(unix_ts, delete_marker)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
|
||||
let candidates = vec![
|
||||
@@ -1455,7 +1446,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
|
||||
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_mod_time(10, false)),
|
||||
@@ -1475,382 +1466,6 @@ mod tests {
|
||||
assert_eq!(idx, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 2,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 7,
|
||||
err: None,
|
||||
},
|
||||
];
|
||||
|
||||
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect("equivalent replicas must resolve deterministically");
|
||||
|
||||
assert_eq!(idx, 7);
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
];
|
||||
|
||||
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect_err("divergent version ids must not silently resolve to the higher pool index");
|
||||
|
||||
assert_eq!(err, Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
];
|
||||
|
||||
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect_err("divergent etags must not silently resolve to the higher pool index");
|
||||
|
||||
assert_eq!(err, Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
];
|
||||
|
||||
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect_err("a delete marker tied with a live version must not be masked by the pool index");
|
||||
|
||||
assert_eq!(err, Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
|
||||
let err = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(left),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(right),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect_err("equal-time identity divergence must fail closed");
|
||||
|
||||
assert_eq!(err, Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
|
||||
let mut data_dir = base.clone();
|
||||
data_dir.data_dir = Some(Uuid::from_u128(2));
|
||||
assert_equal_time_identity_conflict(base.clone(), data_dir);
|
||||
|
||||
let mut size = base.clone();
|
||||
size.size = 1;
|
||||
assert_equal_time_identity_conflict(base.clone(), size);
|
||||
|
||||
let mut actual_size = base.clone();
|
||||
actual_size.actual_size = 1;
|
||||
assert_equal_time_identity_conflict(base.clone(), actual_size);
|
||||
|
||||
let mut checksum = base.clone();
|
||||
checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
|
||||
assert_equal_time_identity_conflict(base.clone(), checksum);
|
||||
|
||||
let mut parts = base.clone();
|
||||
parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
etag: "part-etag".to_string(),
|
||||
number: 1,
|
||||
size: 1,
|
||||
..Default::default()
|
||||
}]);
|
||||
assert_equal_time_identity_conflict(base.clone(), parts);
|
||||
|
||||
let mut transition = base;
|
||||
transition.transitioned_object.tier = "tier-a".to_string();
|
||||
assert_equal_time_identity_conflict(
|
||||
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
|
||||
transition,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
let mut rustfs_alias = base.clone();
|
||||
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"x-rustfs-internal-compression".to_string(),
|
||||
"zstd".to_string(),
|
||||
)]));
|
||||
let mut minio_alias = base.clone();
|
||||
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"X-MINIO-INTERNAL-COMPRESSION".to_string(),
|
||||
"zstd".to_string(),
|
||||
)]));
|
||||
|
||||
let (_, idx) = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(rustfs_alias),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(minio_alias),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect("same-value internal aliases should resolve");
|
||||
assert_eq!(idx, 1);
|
||||
|
||||
let mut dual_alias = base.clone();
|
||||
dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
|
||||
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
|
||||
("x-minio-internal-compression".to_string(), "zstd".to_string()),
|
||||
]));
|
||||
let mut single_alias = base;
|
||||
single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"x-rustfs-internal-compression".to_string(),
|
||||
"zstd".to_string(),
|
||||
)]));
|
||||
|
||||
let (_, idx) = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(dual_alias),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(single_alias),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect("dual-key and single-key internal metadata should resolve");
|
||||
assert_eq!(idx, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
let mut rustfs_alias = base.clone();
|
||||
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"x-rustfs-internal-compression".to_string(),
|
||||
"zstd".to_string(),
|
||||
)]));
|
||||
let mut minio_alias = base;
|
||||
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"x-minio-internal-compression".to_string(),
|
||||
"snappy".to_string(),
|
||||
)]));
|
||||
|
||||
assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
|
||||
for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
let mut rustfs_alias = base.clone();
|
||||
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
format!(
|
||||
"X-RUSTFS-INTERNAL-{}{suffix}",
|
||||
suffix_prefix.to_uppercase(),
|
||||
suffix = "arn:aws:s3:::Bucket"
|
||||
),
|
||||
"value".to_string(),
|
||||
)]));
|
||||
let mut minio_alias = base.clone();
|
||||
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
|
||||
"value".to_string(),
|
||||
)]));
|
||||
|
||||
let (_, idx) = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(rustfs_alias.clone()),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(minio_alias),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect("dynamic internal aliases with the same target should resolve");
|
||||
assert_eq!(idx, 1);
|
||||
|
||||
let mut different_target_case = base;
|
||||
different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
|
||||
"value".to_string(),
|
||||
)]));
|
||||
|
||||
assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
let mut first = base.clone();
|
||||
first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
|
||||
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
|
||||
("x-minio-internal-compression".to_string(), "snappy".to_string()),
|
||||
]));
|
||||
let mut second = base;
|
||||
second.user_defined = first.user_defined.clone();
|
||||
|
||||
assert_equal_time_identity_conflict(first, second);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
|
||||
let mut replication = base.clone();
|
||||
replication.replication_status_internal = Some("PENDING".to_string());
|
||||
replication.replication_status = ReplicationStatusType::Pending;
|
||||
assert_equal_time_identity_conflict(base.clone(), replication);
|
||||
|
||||
let mut purge = base.clone();
|
||||
purge.version_purge_status_internal = Some("PENDING".to_string());
|
||||
purge.version_purge_status = VersionPurgeStatusType::Pending;
|
||||
assert_equal_time_identity_conflict(base.clone(), purge);
|
||||
|
||||
let mut decision = base;
|
||||
decision.replication_decision = "replicate".to_string();
|
||||
assert_equal_time_identity_conflict(
|
||||
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
|
||||
decision,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
|
||||
let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
without_mod_time.mod_time = None;
|
||||
let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
|
||||
assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
|
||||
let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
|
||||
let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
|
||||
older.data_dir = Some(Uuid::from_u128(2));
|
||||
|
||||
let (info, idx) = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(latest),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(older),
|
||||
idx: 9,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect("older identity divergence must not affect the latest candidate");
|
||||
|
||||
assert_eq!(idx, 0);
|
||||
assert_eq!(
|
||||
info.mod_time,
|
||||
Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: None,
|
||||
idx: 1,
|
||||
err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
|
||||
},
|
||||
];
|
||||
|
||||
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect("not-found pools must not block resolution of found candidates");
|
||||
|
||||
assert_eq!(idx, 0);
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
|
||||
let err = resolve_latest_object_info_candidates(
|
||||
|
||||
@@ -12,14 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
|
||||
strip_internal_prefix_preserving_case,
|
||||
};
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
@@ -77,7 +73,7 @@ pub(super) fn resolve_rebalance_delete_from_all_pools_result(
|
||||
object: &str,
|
||||
) -> Result<ObjectInfo> {
|
||||
result.map_err(|err| {
|
||||
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
|
||||
if err == Error::PreconditionFailed {
|
||||
err
|
||||
} else {
|
||||
Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}"))
|
||||
@@ -90,7 +86,7 @@ fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
|
||||
}
|
||||
|
||||
fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
|
||||
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
|
||||
if err == Error::PreconditionFailed {
|
||||
err
|
||||
} else {
|
||||
Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
|
||||
@@ -141,158 +137,37 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p
|
||||
))
|
||||
}
|
||||
|
||||
fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option<OffsetDateTime> {
|
||||
candidate
|
||||
.info
|
||||
.as_ref()
|
||||
.map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
||||
}
|
||||
|
||||
fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
|
||||
left.transition_version_state == right.transition_version_state
|
||||
&& left.transitioned_object.name == right.transitioned_object.name
|
||||
&& left.transitioned_object.version_id == right.transitioned_object.version_id
|
||||
&& left.transitioned_object.tier == right.transitioned_object.tier
|
||||
&& left.transitioned_object.free_version == right.transitioned_object.free_version
|
||||
&& left.transitioned_object.status == right.transitioned_object.status
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq)]
|
||||
struct LatestUserDefinedIdentity {
|
||||
internal: HashMap<String, String>,
|
||||
other: HashMap<String, String>,
|
||||
}
|
||||
|
||||
fn normalize_internal_identity_suffix(key: &str) -> Option<String> {
|
||||
let suffix = strip_internal_prefix_preserving_case(key)?;
|
||||
|
||||
for dynamic_prefix in [
|
||||
SUFFIX_REPLICATION_RESET_ARN_PREFIX,
|
||||
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
|
||||
] {
|
||||
let prefix_len = dynamic_prefix.len();
|
||||
if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..))
|
||||
&& prefix.eq_ignore_ascii_case(dynamic_prefix)
|
||||
{
|
||||
return Some(format!("{dynamic_prefix}{remainder}"));
|
||||
}
|
||||
}
|
||||
|
||||
Some(suffix.to_lowercase())
|
||||
}
|
||||
|
||||
fn normalize_user_defined_identity(user_defined: &HashMap<String, String>) -> Option<LatestUserDefinedIdentity> {
|
||||
let mut identity = LatestUserDefinedIdentity {
|
||||
internal: HashMap::with_capacity(user_defined.len()),
|
||||
other: HashMap::with_capacity(user_defined.len()),
|
||||
};
|
||||
|
||||
for (key, value) in user_defined {
|
||||
if let Some(suffix) = normalize_internal_identity_suffix(key) {
|
||||
if identity
|
||||
.internal
|
||||
.insert(suffix, value.clone())
|
||||
.is_some_and(|previous| previous != *value)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
} else {
|
||||
identity.other.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Some(identity)
|
||||
}
|
||||
|
||||
fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
|
||||
match (
|
||||
normalize_user_defined_identity(&left.user_defined),
|
||||
normalize_user_defined_identity(&right.user_defined),
|
||||
) {
|
||||
(Some(left), Some(right)) => left == right,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
|
||||
/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
|
||||
/// differ for the same object version while the selected winner still carries
|
||||
/// the chosen pool's layout. `put_object_reader` is also intentionally
|
||||
/// excluded because it is a transient request handle that `ObjectInfo::clone`
|
||||
/// drops. Every other ObjectInfo field is part of the production-visible
|
||||
/// identity and must agree before the pool index can provide a deterministic
|
||||
/// tie-break.
|
||||
fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
|
||||
left.bucket == right.bucket
|
||||
&& left.name == right.name
|
||||
&& left.storage_class == right.storage_class
|
||||
&& left.mod_time == right.mod_time
|
||||
&& left.size == right.size
|
||||
&& left.actual_size == right.actual_size
|
||||
&& left.is_dir == right.is_dir
|
||||
&& same_user_defined_identity(left, right)
|
||||
&& left.user_tags == right.user_tags
|
||||
&& left.version_id == right.version_id
|
||||
&& left.data_dir == right.data_dir
|
||||
&& left.delete_marker == right.delete_marker
|
||||
&& same_transition_identity(left, right)
|
||||
&& left.restore_ongoing == right.restore_ongoing
|
||||
&& left.restore_expires == right.restore_expires
|
||||
&& left.parts == right.parts
|
||||
&& left.is_latest == right.is_latest
|
||||
&& left.content_type == right.content_type
|
||||
&& left.content_encoding == right.content_encoding
|
||||
&& left.expires == right.expires
|
||||
&& left.num_versions == right.num_versions
|
||||
&& left.successor_mod_time == right.successor_mod_time
|
||||
&& left.etag == right.etag
|
||||
&& left.inlined == right.inlined
|
||||
&& left.metadata_only == right.metadata_only
|
||||
&& left.version_only == right.version_only
|
||||
&& left.replication_status_internal == right.replication_status_internal
|
||||
&& left.replication_status == right.replication_status
|
||||
&& left.version_purge_status_internal == right.version_purge_status_internal
|
||||
&& left.version_purge_status == right.version_purge_status
|
||||
&& left.replication_decision == right.replication_decision
|
||||
&& left.checksum == right.checksum
|
||||
}
|
||||
|
||||
pub(super) fn resolve_latest_object_info_candidates(
|
||||
candidates: Vec<LatestObjectInfoCandidate>,
|
||||
mut candidates: Vec<LatestObjectInfoCandidate>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max();
|
||||
|
||||
if let Some(latest_mod_time) = latest_mod_time {
|
||||
let mut latest_candidates = candidates
|
||||
.into_iter()
|
||||
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
|
||||
|
||||
let Some(winner) = latest_candidates.first() else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
};
|
||||
let Some(winner_info) = winner.info.as_ref() else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
candidates.sort_by(|a, b| {
|
||||
let a_mod = if let Some(info) = &a.info {
|
||||
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
} else {
|
||||
OffsetDateTime::UNIX_EPOCH
|
||||
};
|
||||
|
||||
if latest_candidates.iter().skip(1).any(|candidate| {
|
||||
candidate
|
||||
.info
|
||||
.as_ref()
|
||||
.is_none_or(|info| !same_latest_object_info_identity(winner_info, info))
|
||||
}) {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
let b_mod = if let Some(info) = &b.info {
|
||||
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
} else {
|
||||
OffsetDateTime::UNIX_EPOCH
|
||||
};
|
||||
|
||||
if a_mod == b_mod {
|
||||
return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less };
|
||||
}
|
||||
|
||||
return Ok((winner_info.clone(), winner.idx));
|
||||
}
|
||||
b_mod.cmp(&a_mod)
|
||||
});
|
||||
|
||||
for candidate in candidates {
|
||||
if let Some(info) = candidate.info {
|
||||
return Ok((info, candidate.idx));
|
||||
}
|
||||
|
||||
if let Some(err) = candidate.err
|
||||
&& !is_err_object_not_found(&err)
|
||||
&& !is_err_version_not_found(&err)
|
||||
@@ -316,18 +191,6 @@ mod tests {
|
||||
assert_eq!(err, Error::PreconditionFailed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_result_preserves_prefix_access_denied() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_result(
|
||||
Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("prefix access denial should remain structured");
|
||||
|
||||
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_pool_result_preserves_precondition_failed() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_results(
|
||||
@@ -342,19 +205,4 @@ mod tests {
|
||||
|
||||
assert_eq!(err, Error::PreconditionFailed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_pool_result_preserves_prefix_access_denied() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_results(
|
||||
vec![RebalanceDeletePoolResult {
|
||||
pool_idx: 0,
|
||||
result: Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
|
||||
}],
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("prefix access denial should remain structured");
|
||||
|
||||
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,13 +54,6 @@ pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_versio
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE: &str = "batch_read_version_request_encode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE: &str = "batch_read_version_request_decode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ: &str = "batch_read_version_disk_read";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch_read_version_response_json_encode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode";
|
||||
|
||||
const OPERATION_LABEL: &str = "operation";
|
||||
const BACKEND_LABEL: &str = "backend";
|
||||
|
||||
@@ -689,14 +689,6 @@ pub struct ScannerMetrics {
|
||||
pub cycle_max_objects: u64,
|
||||
#[serde(rename = "cycle_max_directories", default)]
|
||||
pub cycle_max_directories: u64,
|
||||
#[serde(rename = "cycle_timeout_total", default)]
|
||||
pub cycle_timeout_total: u64,
|
||||
#[serde(rename = "cycle_recovery_required_total", default)]
|
||||
pub cycle_recovery_required_total: u64,
|
||||
#[serde(rename = "cycle_last_progress_age", default)]
|
||||
pub cycle_last_progress_age: u64,
|
||||
#[serde(rename = "leader_lease_without_progress", default)]
|
||||
pub leader_lease_without_progress: bool,
|
||||
#[serde(rename = "bitrot_cycle_enabled", default)]
|
||||
pub bitrot_cycle_enabled: bool,
|
||||
#[serde(rename = "bitrot_cycle_seconds", default)]
|
||||
@@ -772,8 +764,6 @@ impl ScannerMetrics {
|
||||
self.cycle_max_duration_seconds = other.cycle_max_duration_seconds;
|
||||
self.cycle_max_objects = other.cycle_max_objects;
|
||||
self.cycle_max_directories = other.cycle_max_directories;
|
||||
self.cycle_last_progress_age = other.cycle_last_progress_age;
|
||||
self.leader_lease_without_progress = other.leader_lease_without_progress;
|
||||
self.bitrot_cycle_enabled = other.bitrot_cycle_enabled;
|
||||
self.bitrot_cycle_seconds = other.bitrot_cycle_seconds;
|
||||
}
|
||||
@@ -867,12 +857,6 @@ impl ScannerMetrics {
|
||||
.saturating_add(other.last_cycle_replication_checks);
|
||||
self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves);
|
||||
self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles);
|
||||
self.cycle_timeout_total = self.cycle_timeout_total.saturating_add(other.cycle_timeout_total);
|
||||
self.cycle_recovery_required_total = self
|
||||
.cycle_recovery_required_total
|
||||
.saturating_add(other.cycle_recovery_required_total);
|
||||
self.cycle_last_progress_age = self.cycle_last_progress_age.max(other.cycle_last_progress_age);
|
||||
self.leader_lease_without_progress |= other.leader_lease_without_progress;
|
||||
self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles);
|
||||
self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown);
|
||||
self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime);
|
||||
|
||||
@@ -195,8 +195,7 @@ pub struct BucketPolicyArgs<'a> {
|
||||
#[derive(Serialize, Deserialize, Clone, Default, Debug)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BucketPolicy {
|
||||
// RUSTFS_COMPAT_TODO(rustfs-6339): accept bucket policies persisted with the legacy "ID" key. Remove after migration tooling rewrites every retained legacy bucket policy.
|
||||
#[serde(default, rename = "Id", alias = "ID", skip_serializing_if = "ID::is_empty")]
|
||||
#[serde(default, rename = "Id", skip_serializing_if = "ID::is_empty")]
|
||||
pub id: ID,
|
||||
#[serde(rename = "Version")]
|
||||
pub version: String,
|
||||
@@ -2787,7 +2786,7 @@ mod test {
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse");
|
||||
|
||||
// Verify empty fields are omitted
|
||||
assert!(parsed.get("Id").is_none(), "Empty ID should be omitted");
|
||||
assert!(!parsed.as_object().unwrap().contains_key("ID"), "Empty ID should be omitted");
|
||||
|
||||
let statement = &parsed["Statement"][0];
|
||||
assert!(!statement.as_object().unwrap().contains_key("Sid"), "Empty Sid should be omitted");
|
||||
@@ -2810,43 +2809,6 @@ mod test {
|
||||
assert_eq!(statement["Principal"]["AWS"], "*");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_policy_deserializes_legacy_id() {
|
||||
let legacy_policy = br#"{"ID":"","Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"NotAction":[],"Resource":["arn:aws:s3:::bucket/*"],"NotResource":[],"Condition":{}}]}"#;
|
||||
|
||||
let policy: BucketPolicy =
|
||||
serde_json::from_slice(legacy_policy).expect("bucket policy with legacy ID should deserialize");
|
||||
assert!(policy.id.is_empty());
|
||||
policy.is_valid().expect("legacy bucket policy should remain valid");
|
||||
|
||||
let policy: BucketPolicy = serde_json::from_str(r#"{"ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#)
|
||||
.expect("non-empty legacy ID should deserialize");
|
||||
assert_eq!(policy.id.0, "legacy-policy");
|
||||
|
||||
let serialized = serde_json::to_value(&policy).expect("bucket policy should serialize");
|
||||
assert_eq!(serialized["Id"], "legacy-policy");
|
||||
assert!(serialized.get("ID").is_none(), "legacy ID spelling should not be serialized");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_policy_legacy_id_alias_remains_strict() {
|
||||
let unknown_field = r#"{"Version":"2012-10-17","Statement":[],"Unexpected":true}"#;
|
||||
let error =
|
||||
serde_json::from_str::<BucketPolicy>(unknown_field).expect_err("unrelated unknown fields should remain rejected");
|
||||
assert!(
|
||||
error.to_string().contains("unknown field `Unexpected`"),
|
||||
"unexpected deserialization error: {error}"
|
||||
);
|
||||
|
||||
let duplicate_id = r#"{"Id":"current-policy","ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#;
|
||||
let error = serde_json::from_str::<BucketPolicy>(duplicate_id)
|
||||
.expect_err("canonical and legacy ID fields should not be accepted together");
|
||||
assert!(
|
||||
error.to_string().contains("duplicate field `Id`"),
|
||||
"unexpected deserialization error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_existing_object_tag_condition_helpers() {
|
||||
let identity_policy = Policy::parse_config(
|
||||
|
||||
@@ -60,9 +60,7 @@ pub const REPLICATION_READ_ONLY_HISTORICAL_FIELDS: &[&str] = &[
|
||||
"Destination.ReplicationTime",
|
||||
];
|
||||
|
||||
// v2: disableProxy moved from unsupported to writable (per-target read-proxy
|
||||
// opt-out is accepted by set-remote-target and the `proxy` update op).
|
||||
pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 2;
|
||||
pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 1;
|
||||
|
||||
pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[
|
||||
"sourcebucket",
|
||||
@@ -85,12 +83,9 @@ pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[
|
||||
// madmin default of 60s); the per-target health-check interval is not
|
||||
// yet applied — the heartbeat keeps its global env-configured interval.
|
||||
"healthCheckDuration",
|
||||
// Per-target read-proxy opt-out, consumed by the proxy-target selector
|
||||
// (contract v2; previously only importable via MinIO bucket-targets.json).
|
||||
"disableProxy",
|
||||
];
|
||||
|
||||
pub const REMOTE_TARGET_UNSUPPORTED_FIELDS: &[&str] = &["edge", "edgeSyncBeforeExpiry"];
|
||||
pub const REMOTE_TARGET_UNSUPPORTED_FIELDS: &[&str] = &["disableProxy", "edge", "edgeSyncBeforeExpiry"];
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ObjectOpts {
|
||||
|
||||
@@ -125,34 +125,6 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
|
||||
}
|
||||
}
|
||||
|
||||
/// Read only the object revision without materializing its body.
|
||||
pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
|
||||
match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(reader) => reader
|
||||
.object_info
|
||||
.etag
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.map(DataUsageCacheRevision::Etag)
|
||||
.ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))),
|
||||
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
|
||||
Ok(DataUsageCacheRevision::Missing)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision,
|
||||
@@ -174,11 +146,6 @@ pub static LEGACY_DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
|
||||
pub static DATA_USAGE_BLOOM_NAME_PATH: LazyLock<String> =
|
||||
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_BLOOM_NAME}"));
|
||||
|
||||
/// Durable companion object for a cycle-state object which cannot be decoded.
|
||||
/// The primary object is deliberately never replaced or deleted by recovery.
|
||||
pub static DATA_USAGE_BLOOM_RECOVERY_PATH: LazyLock<String> =
|
||||
LazyLock::new(|| format!("{}.recovery-required.json", DATA_USAGE_BLOOM_NAME_PATH.as_str()));
|
||||
|
||||
pub static BACKGROUND_HEAL_INFO_PATH: LazyLock<String> =
|
||||
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}.background-heal.json"));
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ impl DataUsageCache {
|
||||
let loaded = Self::load_cache(store.clone(), name).await?;
|
||||
let backup = match loaded.backup_revision {
|
||||
Some(revision) => Some(revision),
|
||||
None => match read_config_revision(store, &backup_path).await {
|
||||
None => match Self::revision_for_path(store, &backup_path).await {
|
||||
Ok(revision) => Some(revision),
|
||||
Err(err) => {
|
||||
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
|
||||
@@ -336,6 +336,33 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
async fn revision_for_path<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
|
||||
match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(reader) => reader
|
||||
.object_info
|
||||
.etag
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.map(DataUsageCacheRevision::Etag)
|
||||
.ok_or_else(|| StorageError::other(format!("scanner cache object {path} has no ETag"))),
|
||||
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
|
||||
Ok(DataUsageCacheRevision::Missing)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn cache_save_timeout() -> Duration {
|
||||
crate::runtime_config::scanner_cache_save_timeout()
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::persistence::DataUsageCacheLoadAttempt;
|
||||
use super::*;
|
||||
use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO};
|
||||
use crate::{ScannerGetObjectReader, ScannerPutObjReader};
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
|
||||
use serde_json::Value;
|
||||
use std::io::Cursor;
|
||||
use std::pin::Pin;
|
||||
@@ -1636,7 +1636,7 @@ fn size_recursive_prunes_empty_and_preserves_threshold_replication_stats() {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:threshold".to_string(),
|
||||
ReplicationTargetUsage {
|
||||
ReplicationStats {
|
||||
after_threshold_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
|
||||
@@ -75,10 +75,7 @@ pub use remote_scanner::{
|
||||
};
|
||||
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
|
||||
pub use rustfs_common::last_minute;
|
||||
pub use scanner::{
|
||||
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner,
|
||||
reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest,
|
||||
};
|
||||
pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status, scanner_topology_digest};
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
|
||||
record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,
|
||||
|
||||
@@ -125,10 +125,7 @@ impl Default for ScannerRuntimeConfig {
|
||||
cycle_interval_source: ScannerRuntimeConfigSource::Default,
|
||||
bitrot_cycle: Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS)),
|
||||
bitrot_cycle_source: ScannerRuntimeConfigSource::Default,
|
||||
cycle_budget: ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
|
||||
..Default::default()
|
||||
},
|
||||
cycle_budget: ScannerCycleBudgetConfig::default(),
|
||||
cycle_max_duration_source: ScannerRuntimeConfigSource::Default,
|
||||
cycle_max_objects_source: ScannerRuntimeConfigSource::Default,
|
||||
cycle_max_directories_source: ScannerRuntimeConfigSource::Default,
|
||||
@@ -377,10 +374,7 @@ fn validate_persisted_scanner_runtime_config(config: &ServerConfig) -> Result<()
|
||||
}
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_START_DELAY, "")?;
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE, "")?;
|
||||
if let Some(value) = config_value(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) {
|
||||
let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?;
|
||||
cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)?;
|
||||
}
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)?;
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_OBJECTS, DEFAULT_SCANNER_CYCLE_MAX_OBJECTS)?;
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DIRECTORIES, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES)?;
|
||||
if let Some(value) = config_value(heal_kvs, HEAL_BITROT_CYCLE, DEFAULT_HEAL_BITROT_CYCLE_SECS) {
|
||||
@@ -442,46 +436,19 @@ fn lookup_max_wait(
|
||||
Ok((speed.max_sleep(), speed_source))
|
||||
}
|
||||
|
||||
fn lookup_cycle_duration(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
|
||||
match rustfs_utils::get_env_parse_outcome::<u64>(ENV_SCANNER_CYCLE_MAX_DURATION_SECS) {
|
||||
rustfs_utils::EnvParseOutcome::Parsed(secs) => {
|
||||
return cycle_duration_from_secs(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, secs)
|
||||
.map(|duration| (duration, ScannerRuntimeConfigSource::Env));
|
||||
}
|
||||
rustfs_utils::EnvParseOutcome::Invalid => {
|
||||
// Do not include the raw environment value in the typed error:
|
||||
// deployments occasionally put sensitive material in inherited
|
||||
// environment snapshots. The key still identifies the control.
|
||||
return Err(invalid_value(
|
||||
ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
|
||||
"<invalid>",
|
||||
"expected unsigned integer seconds",
|
||||
));
|
||||
}
|
||||
rustfs_utils::EnvParseOutcome::Absent => {}
|
||||
fn lookup_optional_seconds(
|
||||
kvs: Option<&KVS>,
|
||||
key: &'static str,
|
||||
env_key: &'static str,
|
||||
default: u64,
|
||||
) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
|
||||
if let Some(secs) = rustfs_utils::get_env_opt_u64(env_key) {
|
||||
return Ok((Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Env));
|
||||
}
|
||||
|
||||
if let Some(value) = config_value(kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) {
|
||||
let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?;
|
||||
return cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)
|
||||
.map(|duration| (duration, ScannerRuntimeConfigSource::Config));
|
||||
if let Some(value) = config_value(kvs, key, default) {
|
||||
return parse_config_u64(key, value).map(|secs| (Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Config));
|
||||
}
|
||||
|
||||
Ok((
|
||||
Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
|
||||
ScannerRuntimeConfigSource::Default,
|
||||
))
|
||||
}
|
||||
|
||||
fn cycle_duration_from_secs(key: &'static str, secs: u64) -> Result<Option<Duration>, ScannerRuntimeConfigError> {
|
||||
if secs == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let duration = Duration::from_secs(secs);
|
||||
if std::time::Instant::now().checked_add(duration).is_none() {
|
||||
return Err(invalid_value(key, "<overflow>", "duration exceeds the timer range"));
|
||||
}
|
||||
Ok(Some(duration))
|
||||
Ok((None, ScannerRuntimeConfigSource::Default))
|
||||
}
|
||||
|
||||
fn lookup_start_delay(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
|
||||
@@ -586,7 +553,12 @@ pub(crate) fn lookup_scanner_runtime_config(
|
||||
(speed.cycle_interval(), speed_source)
|
||||
};
|
||||
|
||||
let (cycle_max_duration, cycle_max_duration_source) = lookup_cycle_duration(scanner_kvs)?;
|
||||
let (cycle_max_duration, cycle_max_duration_source) = lookup_optional_seconds(
|
||||
scanner_kvs,
|
||||
SCANNER_CYCLE_MAX_DURATION,
|
||||
ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
|
||||
DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS,
|
||||
)?;
|
||||
let (cycle_max_objects, cycle_max_objects_source) = lookup_count_budget(
|
||||
scanner_kvs,
|
||||
SCANNER_CYCLE_MAX_OBJECTS,
|
||||
@@ -891,10 +863,10 @@ mod tests {
|
||||
use rustfs_config::server_config::{Config as ServerConfig, KVS};
|
||||
use rustfs_config::{
|
||||
DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS,
|
||||
ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY,
|
||||
ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE,
|
||||
SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION,
|
||||
SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
||||
ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED,
|
||||
HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
|
||||
SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
@@ -969,50 +941,6 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_unset_budget_uses_safe_default_but_explicit_zero_is_unbounded() {
|
||||
let config = server_config_with_scanner(&[]);
|
||||
with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
|
||||
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||
assert_eq!(resolved.cycle_budget.max_duration, Some(Duration::from_secs(1800)));
|
||||
assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Default);
|
||||
});
|
||||
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "0")]);
|
||||
with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
|
||||
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||
assert_eq!(resolved.cycle_budget.max_duration, None);
|
||||
assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Config);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cycle_budget_invalid_or_overflow_config_is_rejected() {
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("invalid"), || {
|
||||
let error = lookup_scanner_runtime_config(None).expect_err("invalid duration env must be rejected");
|
||||
assert!(error.to_string().contains(ENV_SCANNER_CYCLE_MAX_DURATION_SECS));
|
||||
assert!(error.to_string().contains("<invalid>"));
|
||||
assert!(!error.to_string().contains(": invalid ("));
|
||||
});
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551616"), || {
|
||||
assert!(lookup_scanner_runtime_config(None).is_err());
|
||||
});
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551615"), || {
|
||||
assert!(lookup_scanner_runtime_config(None).is_err());
|
||||
});
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "not-a-duration")]);
|
||||
assert!(lookup_scanner_runtime_config(Some(&config)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_runtime_config_validation_rejects_overflow_persisted_duration() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "18446744073709551615")]);
|
||||
|
||||
let error = validate_scanner_runtime_config(&config)
|
||||
.expect_err("persisted duration that exceeds the timer range must be rejected");
|
||||
assert!(error.to_string().contains(SCANNER_CYCLE_MAX_DURATION));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_runtime_config_normalizes_persisted_default_speed() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]);
|
||||
|
||||
+63
-281
@@ -20,7 +20,7 @@ use std::sync::{Arc, LazyLock, RwLock};
|
||||
|
||||
use crate::data_usage_define::{
|
||||
BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH,
|
||||
DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision,
|
||||
DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision,
|
||||
};
|
||||
use crate::runtime_config::{
|
||||
ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle,
|
||||
@@ -52,10 +52,11 @@ use rustfs_config::{
|
||||
};
|
||||
use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
|
||||
use rustfs_data_usage::observed_data_usage_is_newer;
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use tokio::sync::{Notify, mpsc};
|
||||
#[cfg(test)]
|
||||
use tokio::sync::Notify;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
@@ -103,13 +104,6 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
|
||||
/// unavailable peer cannot drive a tight retry loop.
|
||||
const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
/// A transient backend outage remains self-healing after the short retry
|
||||
/// budget is exhausted, but the probe is intentionally sparse until storage
|
||||
/// recovers or an operator reset wakes the scanner.
|
||||
const SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
||||
/// Permanent recovery states still get a sparse status probe so a reset that
|
||||
/// races the wait registration cannot leave the scanner asleep forever.
|
||||
const SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
||||
const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1);
|
||||
#[cfg(not(test))]
|
||||
const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
@@ -131,12 +125,6 @@ type ScannerCycleStatePersistTestHook = (u64, Arc<Notify>);
|
||||
static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCycleStatePersistTestHook>>> =
|
||||
LazyLock::new(|| StdMutex::new(None));
|
||||
|
||||
static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
|
||||
pub(super) fn notify_scanner_cycle_recovery_wake() {
|
||||
SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct ScannerCycleStatePersistTestHookGuard;
|
||||
|
||||
@@ -588,21 +576,19 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
tokio::time::sleep(sleep_time).await;
|
||||
}
|
||||
|
||||
let mut transient_backoff = ScannerRetryBackoff::default();
|
||||
let mut recovery_retry_count = 0_u32;
|
||||
loop {
|
||||
if ctx_clone.is_cancelled() {
|
||||
break;
|
||||
}
|
||||
|
||||
let run_result = run_data_scanner_with_maintenance_state(
|
||||
if let Err(e) = run_data_scanner_with_maintenance_state(
|
||||
ctx_clone.clone(),
|
||||
storeapi_clone.clone(),
|
||||
startup_features,
|
||||
startup_maintenance_generation,
|
||||
)
|
||||
.await;
|
||||
if let Err(e) = &run_result {
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
@@ -613,52 +599,11 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
"Scanner runtime iteration failed"
|
||||
);
|
||||
}
|
||||
let recovery_status = scanner_cycle_recovery_status();
|
||||
if recovery_status.retryable {
|
||||
recovery_retry_count = recovery_retry_count.saturating_add(1);
|
||||
let _ = record_scanner_cycle_recovery_retry(recovery_retry_count);
|
||||
} else {
|
||||
recovery_retry_count = 0;
|
||||
}
|
||||
|
||||
let recovery_status = scanner_cycle_recovery_status();
|
||||
if recovery_status.state == "paused" {
|
||||
transient_backoff.record_retryable_cycle(false);
|
||||
tokio::select! {
|
||||
_ = ctx_clone.cancelled() => break,
|
||||
_ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
|
||||
_ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {},
|
||||
}
|
||||
recovery_retry_count = 0;
|
||||
continue;
|
||||
}
|
||||
if !recovery_status.retryable
|
||||
&& matches!(recovery_status.state.as_str(), "blocked" | "recovery-required" | "cleanup-pending")
|
||||
{
|
||||
transient_backoff.record_retryable_cycle(false);
|
||||
tokio::select! {
|
||||
_ = ctx_clone.cancelled() => break,
|
||||
_ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
|
||||
_ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL) => {},
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let retry_delay = if recovery_status.retryable || run_result.is_err() {
|
||||
transient_backoff.record_retryable_cycle(true);
|
||||
transient_backoff
|
||||
.retry_interval(scanner_cycle_interval())
|
||||
.unwrap_or(SCANNER_RETRY_BASE_INTERVAL)
|
||||
} else {
|
||||
transient_backoff.record_retryable_cycle(false);
|
||||
randomized_cycle_delay()
|
||||
};
|
||||
// Backoff before retrying after lock contention or scanner-level failures.
|
||||
// Keep this cancellation-aware so shutdown is not delayed by backoff sleep.
|
||||
tokio::select! {
|
||||
_ = ctx_clone.cancelled() => break,
|
||||
_ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
|
||||
_ = tokio::time::sleep(retry_delay) => {}
|
||||
_ = tokio::time::sleep(randomized_cycle_delay()) => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1038,116 +983,20 @@ fn data_usage_persist_timeout() -> Duration {
|
||||
DataUsageCache::persistence_timeout()
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
#[cfg(test)]
|
||||
const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_millis(50);
|
||||
|
||||
async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<Store>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
cycle_revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: &mut u64,
|
||||
lock_lost: LockLost,
|
||||
) -> bool
|
||||
where
|
||||
Store: ScannerObjectIO,
|
||||
LockLost: Future<Output = ()>,
|
||||
{
|
||||
let fence_ctx = ctx.child_token();
|
||||
let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch);
|
||||
tokio::pin!(claim);
|
||||
tokio::pin!(lock_lost);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut lock_lost => {
|
||||
fence_ctx.cancel();
|
||||
false
|
||||
}
|
||||
result = tokio::time::timeout(SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT, &mut claim) => {
|
||||
result.unwrap_or(false) && !fence_ctx.is_cancelled()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ScannerCycleDeadlineState<'a> {
|
||||
cycle_info: &'a mut CurrentCycle,
|
||||
cycle_revision: &'a mut DataUsageCacheRevision,
|
||||
leader_epoch: &'a mut u64,
|
||||
cycle_budget: &'a ScannerCycleBudget,
|
||||
}
|
||||
|
||||
fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool {
|
||||
!worker_stopped || !cycle_state_persisted || !generation_fenced
|
||||
}
|
||||
|
||||
async fn handle_scanner_cycle_deadline<Store>(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<Store>,
|
||||
state: ScannerCycleDeadlineState<'_>,
|
||||
worker_stopped: bool,
|
||||
guard: &mut NamespaceLockGuard,
|
||||
) where
|
||||
Store: ScannerObjectIO,
|
||||
{
|
||||
let fenced = fence_scanner_epoch_after_cycle_timeout(
|
||||
ctx,
|
||||
storeapi,
|
||||
state.cycle_info,
|
||||
state.cycle_revision,
|
||||
state.leader_epoch,
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
.await;
|
||||
let cycle_state_persisted = state.cycle_budget.cycle_state_persisted();
|
||||
let recovery_required = cycle_timeout_requires_recovery(worker_stopped, cycle_state_persisted, fenced);
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "cycle_timeout",
|
||||
worker_stopped,
|
||||
cycle_state_persisted,
|
||||
generation_fenced = fenced,
|
||||
recovery_required,
|
||||
"Scanner cycle deadline expired; durable cursor/generation fencing completed when possible"
|
||||
);
|
||||
global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
|
||||
// Stop renewing before releasing the lease. A new leader can then claim the
|
||||
// higher persisted generation instead of inheriting the expired worker.
|
||||
guard.release();
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
|
||||
cycle_info.current = 0;
|
||||
global_metrics().clear_current_scan_mode();
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[instrument(skip_all)]
|
||||
#[hotpath::measure]
|
||||
async fn run_data_scanner_cycle(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: &Arc<ECStore>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
cycle_revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
) -> ScannerCycleOutcome {
|
||||
let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config());
|
||||
run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget).await
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
#[hotpath::measure]
|
||||
async fn run_data_scanner_cycle_with_budget(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: &Arc<ECStore>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
cycle_revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
cycle_budget: Arc<ScannerCycleBudget>,
|
||||
) -> ScannerCycleOutcome {
|
||||
let _activity_guard = ScannerActivityGuard::new();
|
||||
if let Err(err) = refresh_scanner_runtime_config_from_global() {
|
||||
@@ -1163,11 +1012,7 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
}
|
||||
let configured_cycle_interval = scanner_cycle_interval();
|
||||
let configured_bitrot_cycle = scanner_bitrot_cycle();
|
||||
let cycle_budget_config = ScannerCycleBudgetConfig {
|
||||
max_duration: cycle_budget.max_duration(),
|
||||
max_objects: cycle_budget.max_objects(),
|
||||
max_directories: cycle_budget.max_directories(),
|
||||
};
|
||||
let cycle_budget_config = scanner_cycle_budget_config();
|
||||
let usage_persist_timeout = data_usage_persist_timeout();
|
||||
global_metrics().record_scanner_cycle_config(
|
||||
configured_cycle_interval,
|
||||
@@ -1238,6 +1083,7 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
||||
|
||||
let done_cycle = Metrics::time(Metric::ScanCycle);
|
||||
let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
|
||||
let scan_result = storeapi
|
||||
.clone()
|
||||
.nsscanner_with_status(
|
||||
@@ -1377,7 +1223,7 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
"Scanner cycle is recovering to a newer durable cache generation"
|
||||
);
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let persisted = persist_required_scanner_cycle_floor(
|
||||
return if persist_required_scanner_cycle_floor(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
@@ -1386,9 +1232,8 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
required_cycle,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await;
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
.await
|
||||
{
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
@@ -1446,7 +1291,7 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
scan_cycle_partial_reason(budget_reason),
|
||||
scan_cycle_partial_source(budget_reason),
|
||||
);
|
||||
let persisted = finalize_partial_scan_cycle(
|
||||
return if finalize_partial_scan_cycle(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
@@ -1454,9 +1299,8 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await;
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
.await
|
||||
{
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
@@ -1531,7 +1375,7 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
);
|
||||
}
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let persisted = finalize_partial_scan_cycle(
|
||||
return if finalize_partial_scan_cycle(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
@@ -1539,9 +1383,8 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await;
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
.await
|
||||
{
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
@@ -1582,7 +1425,6 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
)
|
||||
.await
|
||||
{
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
emit_scan_cycle_superseded(cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Superseded;
|
||||
}
|
||||
@@ -1615,7 +1457,6 @@ async fn run_data_scanner_cycle_with_budget(
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
|
||||
done_cycle();
|
||||
emit_scan_cycle_complete(true, cycle_start.elapsed());
|
||||
@@ -1680,7 +1521,7 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
) -> Result<(), ScannerError> {
|
||||
reset_scanner_cycle_schedule();
|
||||
// Acquire leader lock (write lock) to ensure only one scanner runs
|
||||
let mut guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
|
||||
let guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
|
||||
Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
|
||||
Ok(guard) => {
|
||||
record_scanner_leader_lock_state("acquired");
|
||||
@@ -1765,22 +1606,40 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await;
|
||||
}
|
||||
|
||||
let (mut cycle_info, mut leader_epoch, mut cycle_revision) =
|
||||
match load_scanner_cycle_state_for_startup(storeapi.clone()).await {
|
||||
ScannerCycleStateStartup::Ready {
|
||||
cycle,
|
||||
leader_epoch,
|
||||
revision,
|
||||
} => (cycle, leader_epoch, revision),
|
||||
ScannerCycleStateStartup::Blocked => {
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleStateStartup::Transient(err) => {
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let (buf, mut cycle_revision) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await {
|
||||
Ok((buf, revision)) => (buf.unwrap_or_default(), revision),
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
|
||||
state = "revision_load_failed",
|
||||
error = %err,
|
||||
"Scanner cycle state revision load failed"
|
||||
);
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let (mut cycle_info, mut leader_epoch) = match decode_scanner_cycle_state_for_startup(&buf) {
|
||||
Ok(state) => state,
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
|
||||
state = "cycle_decode_failed",
|
||||
error = %err,
|
||||
"Scanner stopped because persisted cycle state is invalid"
|
||||
);
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let usage_floor = match persisted_usage_floor(storeapi.clone()).await {
|
||||
Ok(floor) => floor,
|
||||
Err(err) => {
|
||||
@@ -1845,49 +1704,13 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
return Ok(());
|
||||
}
|
||||
let cycle_ctx = ctx.child_token();
|
||||
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
|
||||
let initial_outcome = match await_scanner_cycle_with_budget_fence(
|
||||
let initial_outcome = await_scanner_cycle_with_lock_fence(
|
||||
&cycle_ctx,
|
||||
&cycle_budget,
|
||||
run_data_scanner_cycle_with_budget(
|
||||
&cycle_ctx,
|
||||
&storeapi,
|
||||
&mut cycle_info,
|
||||
&mut cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget.clone(),
|
||||
),
|
||||
run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
ScannerCycleWaitOutcome::Completed(outcome) => outcome,
|
||||
ScannerCycleWaitOutcome::LockLost => {
|
||||
record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await;
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Cancelled => {
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
cycle_info: &mut cycle_info,
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
},
|
||||
worker_stopped,
|
||||
&mut guard,
|
||||
)
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
.unwrap_or(ScannerCycleOutcome::Failed);
|
||||
superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
|
||||
deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
|
||||
dirty_usage_generation_seen = dirty_generation_before_cycle;
|
||||
@@ -2093,49 +1916,13 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
}
|
||||
let dirty_generation_before_cycle = dirty_usage_generation();
|
||||
let cycle_ctx = ctx.child_token();
|
||||
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
|
||||
let outcome = match await_scanner_cycle_with_budget_fence(
|
||||
let outcome = await_scanner_cycle_with_lock_fence(
|
||||
&cycle_ctx,
|
||||
&cycle_budget,
|
||||
run_data_scanner_cycle_with_budget(
|
||||
&cycle_ctx,
|
||||
&storeapi,
|
||||
&mut cycle_info,
|
||||
&mut cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget.clone(),
|
||||
),
|
||||
run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
ScannerCycleWaitOutcome::Completed(outcome) => outcome,
|
||||
ScannerCycleWaitOutcome::LockLost => {
|
||||
record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await;
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Cancelled => {
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
cycle_info: &mut cycle_info,
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
},
|
||||
worker_stopped,
|
||||
&mut guard,
|
||||
)
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
.unwrap_or(ScannerCycleOutcome::Failed);
|
||||
superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
|
||||
deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
|
||||
dirty_usage_generation_seen = dirty_generation_before_cycle;
|
||||
@@ -2432,12 +2219,7 @@ pub(crate) use activity::{
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
#[cfg(test)]
|
||||
pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
|
||||
pub use cycle_state::{
|
||||
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, reset_scanner_cycle_recovery, scanner_cycle_recovery_status,
|
||||
};
|
||||
pub(crate) use cycle_state::{
|
||||
current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence, load_scanner_cycle_state_for_startup,
|
||||
};
|
||||
pub(crate) use cycle_state::{current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence};
|
||||
pub use heal_info::{BackgroundHealInfo, read_background_heal_info, save_background_heal_info};
|
||||
pub use usage_store::store_data_usage_in_backend;
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -196,7 +196,7 @@ pub(super) async fn claim_scanner_leadership(
|
||||
if ctx.is_cancelled() {
|
||||
return false;
|
||||
}
|
||||
let Some(claimed_epoch) = persisted_epoch.checked_add(1).filter(|epoch| *epoch < u64::MAX) else {
|
||||
let Some(claimed_epoch) = persisted_epoch.checked_add(1) else {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
|
||||
+14
-1106
File diff suppressed because it is too large
Load Diff
@@ -14,16 +14,17 @@
|
||||
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering},
|
||||
atomic::{AtomicU8, AtomicU64, Ordering},
|
||||
};
|
||||
use tokio::time::{Duration, Instant};
|
||||
use std::time::Instant;
|
||||
|
||||
use tokio::time::Duration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const BUDGET_REASON_NONE: u8 = 0;
|
||||
const BUDGET_REASON_RUNTIME: u8 = 1;
|
||||
const BUDGET_REASON_OBJECTS: u8 = 2;
|
||||
const BUDGET_REASON_DIRECTORIES: u8 = 3;
|
||||
const PROGRESS_CLOCK_SAMPLE_INTERVAL: u64 = 128;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub(crate) struct ScannerCycleBudgetConfig {
|
||||
@@ -62,51 +63,29 @@ pub struct ScannerCycleBudget {
|
||||
token: CancellationToken,
|
||||
reason: Arc<AtomicU8>,
|
||||
started_at: Instant,
|
||||
deadline: Option<Instant>,
|
||||
max_duration: Option<Duration>,
|
||||
max_objects: Option<u64>,
|
||||
max_directories: Option<u64>,
|
||||
track_progress: bool,
|
||||
track_unbounded_counts: bool,
|
||||
objects_scanned: AtomicU64,
|
||||
directories_started: AtomicU64,
|
||||
entries_visited: AtomicU64,
|
||||
last_progress_millis: AtomicU64,
|
||||
cycle_state_persisted: AtomicBool,
|
||||
}
|
||||
|
||||
impl ScannerCycleBudget {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
|
||||
Self::new_inner(parent, config, false, false)
|
||||
Self::new_inner(parent, config, false)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
|
||||
Self::new_inner(parent, config, true, true)
|
||||
Self::new_inner(parent, config, true)
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_runtime_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc<Self> {
|
||||
let track_progress = config.max_duration.is_some();
|
||||
Self::new_inner(parent, config, track_progress, false)
|
||||
}
|
||||
|
||||
fn new_inner(
|
||||
parent: &CancellationToken,
|
||||
config: ScannerCycleBudgetConfig,
|
||||
track_progress: bool,
|
||||
track_unbounded_counts: bool,
|
||||
) -> Arc<Self> {
|
||||
fn new_inner(parent: &CancellationToken, config: ScannerCycleBudgetConfig, track_progress: bool) -> Arc<Self> {
|
||||
let token = parent.child_token();
|
||||
let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE));
|
||||
let started_at = Instant::now();
|
||||
let deadline = config.max_duration.map(|duration| match started_at.checked_add(duration) {
|
||||
Some(deadline) => deadline,
|
||||
// Runtime config rejects this range, but keep programmatic callers
|
||||
// fail-closed instead of panicking or silently disabling the wall clock.
|
||||
None => started_at,
|
||||
});
|
||||
|
||||
if let Some(deadline) = deadline {
|
||||
if let Some(duration) = config.max_duration {
|
||||
let parent = parent.clone();
|
||||
let token_wait = token.clone();
|
||||
let token_cancel = token.clone();
|
||||
@@ -115,7 +94,7 @@ impl ScannerCycleBudget {
|
||||
tokio::select! {
|
||||
_ = parent.cancelled() => {}
|
||||
_ = token_wait.cancelled() => {}
|
||||
_ = tokio::time::sleep_until(deadline) => {
|
||||
_ = tokio::time::sleep(duration) => {
|
||||
Self::cancel_for_reason(&reason, &token_cancel, ScannerCycleBudgetReason::Runtime);
|
||||
}
|
||||
}
|
||||
@@ -125,18 +104,14 @@ impl ScannerCycleBudget {
|
||||
Arc::new(Self {
|
||||
token,
|
||||
reason,
|
||||
started_at,
|
||||
deadline,
|
||||
started_at: Instant::now(),
|
||||
max_duration: config.max_duration,
|
||||
max_objects: config.max_objects,
|
||||
max_directories: config.max_directories,
|
||||
track_progress,
|
||||
track_unbounded_counts,
|
||||
objects_scanned: AtomicU64::new(0),
|
||||
directories_started: AtomicU64::new(0),
|
||||
entries_visited: AtomicU64::new(0),
|
||||
last_progress_millis: AtomicU64::new(0),
|
||||
cycle_state_persisted: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -156,14 +131,6 @@ impl ScannerCycleBudget {
|
||||
self.max_duration
|
||||
}
|
||||
|
||||
pub(crate) fn deadline(&self) -> Option<Instant> {
|
||||
self.deadline
|
||||
}
|
||||
|
||||
pub(crate) fn cancel_for_runtime(&self) {
|
||||
self.cancel_for(ScannerCycleBudgetReason::Runtime);
|
||||
}
|
||||
|
||||
pub(crate) fn max_objects(&self) -> Option<u64> {
|
||||
self.max_objects
|
||||
}
|
||||
@@ -206,43 +173,15 @@ impl ScannerCycleBudget {
|
||||
self.entries_visited.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub(crate) fn mark_cycle_state_persisted(&self) {
|
||||
self.cycle_state_persisted.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn cycle_state_persisted(&self) -> bool {
|
||||
self.cycle_state_persisted.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn progress_age(&self) -> Duration {
|
||||
let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
let last_progress = self.last_progress_millis.load(Ordering::Relaxed);
|
||||
Duration::from_millis(elapsed_millis.saturating_sub(last_progress))
|
||||
}
|
||||
|
||||
fn record_progress_sample(&self, event: u64) {
|
||||
// Clock reads are sampled at batch/count boundaries; the scanner's
|
||||
// per-object path does not add a second progress atomic.
|
||||
if event == 0 || (event != 1 && !event.is_multiple_of(PROGRESS_CLOCK_SAMPLE_INTERVAL)) {
|
||||
return;
|
||||
}
|
||||
let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
self.last_progress_millis.store(elapsed_millis, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn record_entries_visited(&self, entries_visited: u64) {
|
||||
if self.track_progress {
|
||||
let entries = saturating_fetch_add(&self.entries_visited, entries_visited);
|
||||
self.record_progress_sample(entries);
|
||||
saturating_fetch_add(&self.entries_visited, entries_visited);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) {
|
||||
if self.track_progress || self.max_objects.is_some() {
|
||||
let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned);
|
||||
if self.track_progress {
|
||||
self.record_progress_sample(objects);
|
||||
}
|
||||
if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
|
||||
self.cancel_for(ScannerCycleBudgetReason::Objects);
|
||||
}
|
||||
@@ -250,12 +189,9 @@ impl ScannerCycleBudget {
|
||||
|
||||
if self.track_progress || self.max_directories.is_some() {
|
||||
let directories = saturating_fetch_add(&self.directories_started, directories_started);
|
||||
if self.track_progress {
|
||||
self.record_progress_sample(directories);
|
||||
}
|
||||
if self
|
||||
.max_directories
|
||||
.is_some_and(|max_directories| directory_budget_exhausted(directories, max_directories))
|
||||
.is_some_and(|max_directories| directories > max_directories)
|
||||
{
|
||||
self.cancel_for(ScannerCycleBudgetReason::Directories);
|
||||
}
|
||||
@@ -271,17 +207,14 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn try_start_directory(&self) -> bool {
|
||||
if self.max_directories.is_none() && !self.track_unbounded_counts {
|
||||
if !self.track_progress && self.max_directories.is_none() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let directories = saturating_fetch_add(&self.directories_started, 1);
|
||||
if self.track_progress {
|
||||
self.record_progress_sample(directories);
|
||||
}
|
||||
if self
|
||||
.max_directories
|
||||
.is_some_and(|max_directories| directory_budget_exhausted(directories, max_directories))
|
||||
.is_some_and(|max_directories| directories > max_directories)
|
||||
{
|
||||
self.cancel_for(ScannerCycleBudgetReason::Directories);
|
||||
return false;
|
||||
@@ -291,14 +224,11 @@ impl ScannerCycleBudget {
|
||||
}
|
||||
|
||||
pub(crate) fn record_object_scanned(&self) {
|
||||
if self.max_objects.is_none() && !self.track_unbounded_counts {
|
||||
if !self.track_progress && self.max_objects.is_none() {
|
||||
return;
|
||||
}
|
||||
|
||||
let objects = saturating_fetch_add(&self.objects_scanned, 1);
|
||||
if self.track_progress {
|
||||
self.record_progress_sample(objects);
|
||||
}
|
||||
if self.max_objects.is_some_and(|max_objects| objects >= max_objects) {
|
||||
self.cancel_for(ScannerCycleBudgetReason::Objects);
|
||||
}
|
||||
@@ -329,13 +259,6 @@ fn saturating_fetch_add(value: &AtomicU64, delta: u64) -> u64 {
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_budget_exhausted(directories: u64, max_directories: u64) -> bool {
|
||||
// Saturation hides a remote max+1 update when the configured limit is the
|
||||
// largest representable counter. Treat that boundary as exhausted rather
|
||||
// than allowing work to continue indefinitely.
|
||||
directories > max_directories || (directories == u64::MAX && max_directories == u64::MAX)
|
||||
}
|
||||
|
||||
impl Drop for ScannerCycleBudget {
|
||||
fn drop(&mut self) {
|
||||
self.token.cancel();
|
||||
@@ -478,35 +401,6 @@ mod tests {
|
||||
assert_eq!(directory_budget.reason(), Some(ScannerCycleBudgetReason::Directories));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_budget_fails_closed_when_progress_saturates() {
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_directories: Some(u64::MAX),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
budget.record_remote_progress(0, u64::MAX);
|
||||
|
||||
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Directories));
|
||||
assert!(budget.token().is_cancelled());
|
||||
|
||||
let local_budget = ScannerCycleBudget::new(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_directories: Some(u64::MAX),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
local_budget.record_remote_progress(0, u64::MAX - 1);
|
||||
assert!(!local_budget.budget_elapsed());
|
||||
assert!(!local_budget.try_start_directory());
|
||||
assert_eq!(local_budget.reason(), Some(ScannerCycleBudgetReason::Directories));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_progress_tracking_counts_unbounded_remote_work_without_cancelling() {
|
||||
let parent = CancellationToken::new();
|
||||
@@ -567,29 +461,4 @@ mod tests {
|
||||
assert!(object_limited.requires_serial_progress_accounting());
|
||||
assert!(directory_limited.requires_serial_progress_accounting());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn progress_age_uses_virtual_time_and_sampled_progress() {
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_runtime_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_secs(60)),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
|
||||
tokio::time::advance(Duration::from_secs(5)).await;
|
||||
assert_eq!(budget.progress_age(), Duration::from_secs(5));
|
||||
budget.record_entries_visited(1);
|
||||
assert_eq!(budget.progress_age(), Duration::ZERO);
|
||||
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
for _ in 0..126 {
|
||||
budget.record_entries_visited(1);
|
||||
}
|
||||
assert_eq!(budget.progress_age(), Duration::from_secs(2));
|
||||
budget.record_entries_visited(1);
|
||||
assert_eq!(budget.progress_age(), Duration::ZERO);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,6 @@ use time::OffsetDateTime;
|
||||
use tokio::sync::{Mutex, Notify, Semaphore, mpsc};
|
||||
use tokio::time::Duration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use crate::ScannerObjectInfo as ObjectInfo;
|
||||
|
||||
@@ -314,7 +314,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let ctx_clone = ctx.clone();
|
||||
let completed_bucket_count = Arc::new(AtomicUsize::new(0));
|
||||
let completed_bucket_count_clone = completed_bucket_count.clone();
|
||||
let collect_bucket_results_fut = AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
let collect_bucket_results_fut = tokio::spawn(async move {
|
||||
let mut cancelled = false;
|
||||
|
||||
loop {
|
||||
@@ -333,7 +333,7 @@ impl ScannerIOCache for SetDisks {
|
||||
}
|
||||
}
|
||||
}
|
||||
}));
|
||||
});
|
||||
|
||||
let mut futs = Vec::new();
|
||||
|
||||
@@ -365,7 +365,7 @@ impl ScannerIOCache for SetDisks {
|
||||
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
|
||||
NamespaceScannerWorkerMode::Coordinator => None,
|
||||
};
|
||||
futs.push(AbortOnDropHandle::new(tokio::spawn(async move {
|
||||
futs.push(tokio::spawn(async move {
|
||||
let remote_session_id = uuid::Uuid::new_v4();
|
||||
let mut remote_session_sequence = 0_u64;
|
||||
loop {
|
||||
@@ -1038,7 +1038,7 @@ impl ScannerIOCache for SetDisks {
|
||||
);
|
||||
}
|
||||
}
|
||||
})));
|
||||
}));
|
||||
}
|
||||
drop(bucket_tx);
|
||||
drop(bucket_result_tx);
|
||||
|
||||
@@ -242,7 +242,7 @@ impl ScannerIOCycle for ECStore {
|
||||
results[results_index_clone] = result;
|
||||
}
|
||||
});
|
||||
wait_futs.push(AbortOnDropHandle::new(receiver_fut));
|
||||
wait_futs.push(receiver_fut);
|
||||
|
||||
let scan_plan = ScannerBucketScanPlan {
|
||||
buckets: set_buckets,
|
||||
@@ -318,7 +318,7 @@ impl ScannerIOCycle for ECStore {
|
||||
record_set_scan_failure(&mut first_err, e);
|
||||
}
|
||||
});
|
||||
wait_futs.push(AbortOnDropHandle::new(scanner_fut));
|
||||
wait_futs.push(scanner_fut);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
|
||||
|
||||
const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]);
|
||||
|
||||
@@ -271,7 +271,7 @@ fn completed_data_usage_info_flattens_nested_bucket_entries() {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:target".to_string(),
|
||||
ReplicationTargetUsage {
|
||||
ReplicationStats {
|
||||
replicated_size: 2048,
|
||||
replicated_count: 2,
|
||||
..Default::default()
|
||||
|
||||
@@ -76,7 +76,6 @@ pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOption
|
||||
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use object::DeleteAccounting;
|
||||
pub use object::ObjectLockDeleteOptions;
|
||||
pub use object::{DeletedObject, ObjectToDelete};
|
||||
pub use object::{ExpirationOptions, TransitionedObject};
|
||||
|
||||
@@ -218,17 +218,6 @@ pub struct DeletedObject {
|
||||
pub force_delete_generation: Option<i64>,
|
||||
}
|
||||
|
||||
/// Accounting identity returned by the internal commit-time delete path.
|
||||
///
|
||||
/// This is carried separately from [`DeletedObject`] so adding quota details
|
||||
/// does not change the source shape of the public S3 delete result contract.
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub struct DeleteAccounting {
|
||||
pub size: Option<u64>,
|
||||
pub version_id: Option<Uuid>,
|
||||
pub removed_current_object: bool,
|
||||
}
|
||||
|
||||
impl DeletedObject {
|
||||
pub fn version_purge_status(&self) -> VersionPurgeStatusType {
|
||||
self.replication_state
|
||||
@@ -352,19 +341,6 @@ pub trait ObjectOperations: Send + Sync + fmt::Debug {
|
||||
objects: Vec<Self::ObjectToDelete>,
|
||||
opts: Self::ObjectOptions,
|
||||
) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>);
|
||||
/// Delete objects and optionally return commit-time accounting identities.
|
||||
/// The default preserves the ordinary delete contract for implementations
|
||||
/// that do not expose storage-level accounting details.
|
||||
async fn delete_objects_with_accounting(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: Vec<Self::ObjectToDelete>,
|
||||
opts: Self::ObjectOptions,
|
||||
) -> (Vec<Self::DeletedObject>, Vec<Option<Self::Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
let object_count = objects.len();
|
||||
let (deleted, errors) = self.delete_objects(bucket, objects, opts).await;
|
||||
(deleted, errors, vec![None; object_count])
|
||||
}
|
||||
async fn put_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -268,7 +268,7 @@ where
|
||||
.parse::<T>()
|
||||
.map_err(|_| {
|
||||
log_once(&format!("env_invalid_value:{used_key}"), || {
|
||||
format!("Invalid {} value for {used_key}. Treating as unset.", type_name::<T>())
|
||||
format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::<T>())
|
||||
});
|
||||
})
|
||||
.ok()
|
||||
@@ -570,7 +570,7 @@ where
|
||||
Ok(parsed) => EnvParseOutcome::Parsed(parsed),
|
||||
Err(_) => {
|
||||
log_once(&format!("env_invalid_value:{used_key}"), || {
|
||||
format!("Invalid {} value for {used_key}. Treating as unset.", type_name::<T>())
|
||||
format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::<T>())
|
||||
});
|
||||
EnvParseOutcome::Invalid
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ for later deletion.
|
||||
|
||||
## Open Items
|
||||
|
||||
- `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID".
|
||||
- `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations.
|
||||
- `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2.
|
||||
- `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration.
|
||||
|
||||
@@ -52,7 +52,7 @@ The `/v3/scanner/status` response reports each effective runtime value with a
|
||||
| `scanner.max_wait` | `RUSTFS_SCANNER_MAX_WAIT_SECS` | seconds | preset-derived | Caps one scanner sleep. |
|
||||
| `scanner.cycle` | `RUSTFS_SCANNER_CYCLE` | seconds | preset-derived | Sets the interval between scanner cycles. |
|
||||
| `scanner.start_delay` | `RUSTFS_SCANNER_START_DELAY_SECS` | seconds | unset | Sets startup delay and, for compatibility, the cycle interval when `scanner.cycle` is unset. |
|
||||
| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `1800` | Caps one cycle's runtime. An explicit `0` disables this budget. |
|
||||
| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `0` | Caps one cycle's runtime. `0` disables this budget. |
|
||||
| `scanner.cycle_max_objects` | `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` | objects | `0` | Caps objects processed by one cycle. `0` disables this budget. |
|
||||
| `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` | Caps directories entered by one cycle. `0` disables this budget. |
|
||||
| `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. |
|
||||
@@ -70,21 +70,6 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`,
|
||||
`scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis
|
||||
needs a precise override.
|
||||
|
||||
When the cycle duration control is unset, RustFS uses a finite 1800-second
|
||||
(30-minute) default, matching the scanner benchmark guidance. An explicit `0`
|
||||
preserves the compatibility behavior of an unbounded cycle; object and
|
||||
directory budgets likewise remain unbounded when explicitly set to `0`. Invalid
|
||||
or overflowing duration environment values are configuration errors rather than
|
||||
silent fallback values.
|
||||
|
||||
When a finite deadline expires, RustFS cancels cooperative scanner work and
|
||||
waits only for the existing bounded shutdown window. A non-yielding I/O future
|
||||
is dropped after that window. RustFS then attempts a higher leadership epoch so
|
||||
late cycle, usage, cache, and remote writes from the old generation fail closed.
|
||||
If the worker cannot stop cooperatively, the cycle state was not confirmed
|
||||
durable, or that epoch fence cannot be durably persisted, the scanner reports
|
||||
`recovery-required`; it does not claim an uncooperative cursor was saved.
|
||||
|
||||
An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle
|
||||
cadence: dirty-usage notifications do not bypass that configured interval.
|
||||
The default adaptive policy continues to use dirty-usage notifications to wake
|
||||
@@ -159,10 +144,6 @@ metrics.maintenance_control.primary_control
|
||||
metrics.source_work
|
||||
metrics.replication_repair
|
||||
metrics.scan_checkpoint
|
||||
metrics.cycle_timeout_total
|
||||
metrics.cycle_last_progress_age
|
||||
metrics.leader_lease_without_progress
|
||||
metrics.cycle_recovery_required_total
|
||||
```
|
||||
|
||||
## Reading Pacing Pressure
|
||||
|
||||
+4
-11
@@ -158,11 +158,10 @@ added by backlog#1153 infra-4.
|
||||
|
||||
## Coverage
|
||||
|
||||
Workspace line coverage is measured weekly. Pull requests that touch iam, kms,
|
||||
policy, or crypto also run a non-required, report-only comparison against
|
||||
`.config/coverage-baselines.toml`. During calibration, a regression is recorded
|
||||
in the job summary without failing the job; missing or malformed coverage
|
||||
evidence still fails closed (backlog#1153 infra-6).
|
||||
Line coverage is measured **weekly, not per-PR**, and is non-blocking: it
|
||||
exists for visibility and trend, never as a required check. Per-crate ratchets
|
||||
for the security-critical crates (iam / kms / policy / crypto) build on this
|
||||
baseline later (backlog#1153 infra-6, report-only first).
|
||||
|
||||
- **CI**: `.github/workflows/coverage.yml` runs every Sunday and on manual
|
||||
dispatch: `cargo llvm-cov nextest --workspace --exclude e2e_test` under the
|
||||
@@ -175,12 +174,6 @@ evidence still fails closed (backlog#1153 infra-6).
|
||||
plus the full suite). It prints the same per-crate table via
|
||||
`scripts/coverage_per_crate.py` and writes `target/llvm-cov/lcov.info` and
|
||||
`coverage.json`.
|
||||
- **Security-critical ratchet**: relevant pull requests compare iam / kms /
|
||||
policy / crypto line coverage with the versioned baseline. Drops greater than
|
||||
the configured one-percentage-point calibration threshold are marked
|
||||
`REGRESSION (report-only)`. The weekly summary runs the same comparison so
|
||||
calibration continues even when no relevant pull request is open. Baseline
|
||||
changes require a linked coverage run and a reviewed explanation.
|
||||
- **Trend comparison**: each run's job summary is the weekly per-crate
|
||||
snapshot — open two runs from the Actions history (workflow "coverage") and
|
||||
compare their tables. For line-level diffs, download the two runs'
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user