mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 20:36:38 +00:00
Compare commits
52 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| acc37b49c8 | |||
| 2379bfb5a3 | |||
| 4527b003cc | |||
| c83aa7f974 | |||
| 2d7120460b | |||
| b91845c98c | |||
| 7ba6f8cb33 | |||
| f44b30c61a | |||
| c6590182ed | |||
| 6d85a9c6a8 | |||
| 26e6508b64 | |||
| 51e369be6c | |||
| ddc4120c82 | |||
| 87235ffd28 | |||
| 62c465ecef | |||
| f1b92af4a3 | |||
| 7c1a76dfd9 | |||
| 3ddf1a81ac | |||
| cc412914d5 | |||
| b6f135f0fe | |||
| b07f0a92a8 | |||
| 2fccfdeabe | |||
| 9815694301 | |||
| 0e79106c2f | |||
| 12a9e654b5 | |||
| 6b5e0feef6 | |||
| da90d02c15 | |||
| a930152d5a | |||
| f9d45e41e1 | |||
| 34a52a1e7d | |||
| 04e1ea227a | |||
| fe2d166cd3 | |||
| c687f95260 | |||
| fe66aa509e | |||
| 918686bd4e | |||
| a9cbf82943 | |||
| ee7a82ec1f | |||
| 3229ce05e3 | |||
| 7428f3138c | |||
| cf0a78ce11 | |||
| 7bda7f4825 | |||
| 20a8c443af | |||
| 948ee1fc52 | |||
| ecd779b6f3 | |||
| 619fae2512 | |||
| 4f9d22bda7 | |||
| 1a3be70d98 | |||
| 8679570c2a | |||
| 7143697a5f | |||
| a34310a58f | |||
| 2e60029079 | |||
| 2c3e68ad89 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b
|
||||
sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba
|
||||
sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52
|
||||
sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6
|
||||
|
||||
@@ -36,6 +36,7 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_manual_transition_runbooks.sh
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/s3-tests/test_report_compat.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
|
||||
@@ -78,6 +78,12 @@ test-group = 'embedded-test-ports'
|
||||
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# The durable ILM decommission regressions build isolated multi-pool stores and
|
||||
# deliberately take source or target disks offline while checking fencing.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the bucket-incarnation / lifecycle-fence tests. They drive
|
||||
# init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
|
||||
# OnceLock state that serial_test's #[serial] cannot protect across nextest's
|
||||
@@ -190,6 +196,10 @@ test-group = 'embedded-test-ports'
|
||||
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
|
||||
# too (see the matching default-profile override near the top). No retries.
|
||||
[[profile.ci.overrides]]
|
||||
|
||||
@@ -14,9 +14,10 @@
|
||||
|
||||
name: "Schedule Failure Issue"
|
||||
description: >-
|
||||
Open (or update) a tracking issue when a scheduled workflow run fails.
|
||||
Open (or update) a tracking issue when a scheduled workflow run fails or
|
||||
does not complete normally.
|
||||
Dedupes by workflow name: if an open issue titled
|
||||
"[scheduled-failure] <workflow name>" already exists, the failure is
|
||||
"[scheduled-failure] <workflow name>" already exists, the result is
|
||||
appended as a comment; otherwise a new issue is created. This is the
|
||||
single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
|
||||
|
||||
@@ -38,6 +39,30 @@ inputs:
|
||||
Set to an empty string to skip labeling.
|
||||
required: false
|
||||
default: "infrastructure"
|
||||
source-run-id:
|
||||
description: "Run ID to report. Defaults to the current workflow run."
|
||||
required: false
|
||||
default: ${{ github.run_id }}
|
||||
source-run-attempt:
|
||||
description: "Run attempt to report. Defaults to the current attempt."
|
||||
required: false
|
||||
default: ${{ github.run_attempt }}
|
||||
source-event:
|
||||
description: "Trigger event of the run being reported."
|
||||
required: false
|
||||
default: ${{ github.event_name }}
|
||||
source-ref-name:
|
||||
description: "Ref name of the run being reported."
|
||||
required: false
|
||||
default: ${{ github.ref_name }}
|
||||
source-sha:
|
||||
description: "Commit SHA of the run being reported."
|
||||
required: false
|
||||
default: ${{ github.sha }}
|
||||
details-file:
|
||||
description: "Optional Markdown file appended to the issue body."
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
@@ -48,17 +73,22 @@ runs:
|
||||
GH_TOKEN: ${{ inputs.github-token }}
|
||||
WORKFLOW_NAME: ${{ inputs.workflow-name }}
|
||||
ISSUE_LABEL: ${{ inputs.label }}
|
||||
SOURCE_RUN_ID: ${{ inputs.source-run-id }}
|
||||
SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
|
||||
SOURCE_EVENT: ${{ inputs.source-event }}
|
||||
SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
|
||||
SOURCE_SHA: ${{ inputs.source-sha }}
|
||||
DETAILS_FILE: ${{ inputs.details-file }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
title="[scheduled-failure] ${WORKFLOW_NAME}"
|
||||
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
|
||||
|
||||
# Failed job names for this run attempt. The alert job runs while the
|
||||
# run as a whole is still in progress, so inspect the jobs that have
|
||||
# already completed with a non-success conclusion.
|
||||
# Inspect the reported run attempt. It can be the current in-workflow
|
||||
# failure or a completed run observed by the external watchdog.
|
||||
failed_jobs="$(gh api \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
|
||||
--paginate \
|
||||
--jq '.jobs[]
|
||||
| select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
|
||||
@@ -67,15 +97,26 @@ runs:
|
||||
failed_jobs="- (failed job not recorded yet — see the run page)"
|
||||
fi
|
||||
|
||||
details=""
|
||||
if [ -n "${DETAILS_FILE}" ]; then
|
||||
if [ -f "${DETAILS_FILE}" ]; then
|
||||
details="$(cat "${DETAILS_FILE}")"
|
||||
else
|
||||
details="Details file was not available: \`${DETAILS_FILE}\`"
|
||||
fi
|
||||
fi
|
||||
|
||||
body="$(cat <<EOF
|
||||
Scheduled run of **${WORKFLOW_NAME}** failed.
|
||||
Run of **${WORKFLOW_NAME}** did not complete successfully.
|
||||
|
||||
- Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
|
||||
- Event: \`${GITHUB_EVENT_NAME}\`
|
||||
- Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
|
||||
- Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
|
||||
- Event: \`${SOURCE_EVENT}\`
|
||||
- Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
|
||||
|
||||
Failed jobs:
|
||||
Non-success jobs:
|
||||
${failed_jobs}
|
||||
|
||||
${details}
|
||||
EOF
|
||||
)"
|
||||
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
[
|
||||
{ "workflow": ".github/workflows/audit.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/build.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
|
||||
]
|
||||
@@ -46,7 +46,7 @@ on:
|
||||
# advisory could sit unnoticed for seven days. The check list is unchanged —
|
||||
# splitting it into a light daily advisories-only run and a weekly full run
|
||||
# would create runs where sources/bans/licenses go unverified.
|
||||
- cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
|
||||
- cron: '23 3 * * *' # Daily 03:23 UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -52,7 +52,7 @@ on:
|
||||
- ".dockerignore"
|
||||
- "flake.lock"
|
||||
schedule:
|
||||
- cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
|
||||
- cron: "13 1 * * 0" # Weekly on Sunday 01:13 UTC
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_docker:
|
||||
@@ -1032,3 +1032,23 @@ jobs:
|
||||
|
||||
echo "🎉 Released $TAG successfully!"
|
||||
echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -94,6 +94,9 @@ concurrency:
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
# Swatinem/rust-cache hashes every RUST* variable. Keep this aligned with
|
||||
# ci.yml or the writer and readers use disjoint cache keys.
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary,
|
||||
@@ -101,7 +104,7 @@ jobs:
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -191,7 +194,7 @@ jobs:
|
||||
warm-ci-feat-rio:
|
||||
name: Warm ci-feat-rio
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -219,7 +222,7 @@ jobs:
|
||||
warm-ci-feat-proto:
|
||||
name: Warm ci-feat-proto
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
|
||||
@@ -126,7 +126,10 @@ jobs:
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
|
||||
@@ -59,7 +59,7 @@ on:
|
||||
merge_group:
|
||||
types: [ checks_requested ]
|
||||
schedule:
|
||||
- cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
|
||||
- cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
@@ -161,7 +161,10 @@ jobs:
|
||||
run: ./scripts/check_embedded_secrets.sh
|
||||
|
||||
- name: Check test wiring
|
||||
run: python3 ./scripts/check_test_wiring.py
|
||||
run: |
|
||||
python3 ./scripts/check_test_wiring.py --self-test
|
||||
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
|
||||
python3 ./scripts/check_test_wiring.py
|
||||
|
||||
- name: Check no planning docs committed
|
||||
run: ./scripts/check_no_planning_docs.sh
|
||||
@@ -400,7 +403,7 @@ jobs:
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 45
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -440,7 +443,7 @@ jobs:
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
@@ -470,7 +473,7 @@ jobs:
|
||||
if: github.event_name != 'pull_request' || github.event.action != 'closed'
|
||||
needs: [ quick-checks ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
# On a PR, one failing protocol leg is enough to know the PR is not ready,
|
||||
# so stop the sibling leg instead of paying another ~40 minutes for it.
|
||||
@@ -1032,3 +1035,37 @@ jobs:
|
||||
path: artifacts/s3tests-single/**
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs:
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- test-ilm-integration-serial
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
- build-rustfs-debug-binary-rio-v2
|
||||
- uring-integration
|
||||
- e2e-tests
|
||||
- e2e-full
|
||||
- e2e-tests-rio-v2
|
||||
- s3-implemented-tests
|
||||
- s3-lifecycle-behavior-tests
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -37,7 +37,7 @@ on:
|
||||
# build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
|
||||
# (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
|
||||
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
|
||||
- cron: "0 7 * * 0"
|
||||
- cron: "43 7 * * 0"
|
||||
|
||||
# Only alert-on-failure needs more than read access; it declares its own
|
||||
# job-level `issues: write`.
|
||||
|
||||
@@ -40,7 +40,7 @@ on:
|
||||
schedule:
|
||||
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
|
||||
# stale (01:30) and performance-ab (06:00).
|
||||
- cron: "0 4 * * *"
|
||||
- cron: "29 4 * * *"
|
||||
|
||||
# Only alert-on-failure needs more than read access; it declares its own
|
||||
# job-level `issues: write`.
|
||||
@@ -196,6 +196,9 @@ jobs:
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Verify protocol socket oracle
|
||||
run: ss -tn state CLOSE-WAIT >/dev/null
|
||||
|
||||
# The suite owns fixed protocol ports and serializes its internal cases.
|
||||
- name: Verify protocol e2e membership
|
||||
env:
|
||||
|
||||
@@ -90,10 +90,14 @@ on:
|
||||
description: "Optional pytest -m expression"
|
||||
required: false
|
||||
default: ""
|
||||
testexpr:
|
||||
description: "Optional pytest -k expression"
|
||||
required: false
|
||||
default: ""
|
||||
schedule:
|
||||
# Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
|
||||
# single-node and the 4-node distributed topologies (matrix below).
|
||||
- cron: "0 2 * * 0"
|
||||
- cron: "19 2 * * 0"
|
||||
|
||||
env:
|
||||
# main user
|
||||
@@ -116,6 +120,7 @@ env:
|
||||
XDIST: ${{ github.event.inputs.xdist || '4' }}
|
||||
MAXFAIL: ${{ github.event.inputs.maxfail || '0' }}
|
||||
MARKEXPR: ${{ github.event.inputs.markexpr || '' }}
|
||||
TESTEXPR: ${{ github.event.inputs.testexpr || '' }}
|
||||
S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }}
|
||||
TEST_TIMEOUT: "300"
|
||||
|
||||
@@ -269,14 +274,20 @@ jobs:
|
||||
EOF
|
||||
|
||||
cat > haproxy.cfg <<'EOF'
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
log-format '%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %HM %HP'
|
||||
timeout connect 5s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
|
||||
frontend fe_s3
|
||||
bind *:9000
|
||||
option http-buffer-request
|
||||
default_backend be_s3
|
||||
|
||||
backend be_s3
|
||||
@@ -314,6 +325,7 @@ jobs:
|
||||
XDIST="${XDIST}" \
|
||||
MAXFAIL="${MAXFAIL}" \
|
||||
MARKEXPR="${MARKEXPR}" \
|
||||
TESTEXPR="${TESTEXPR}" \
|
||||
./scripts/s3-tests/run.sh
|
||||
|
||||
- name: Publish compatibility report
|
||||
|
||||
@@ -30,7 +30,7 @@ on:
|
||||
- "Cargo.lock"
|
||||
- ".github/workflows/fuzz.yml"
|
||||
schedule:
|
||||
- cron: "0 2 * * *"
|
||||
- cron: "17 2 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
profile:
|
||||
|
||||
@@ -121,3 +121,21 @@ jobs:
|
||||
cargo nextest run --run-ignored ignored-only --no-tests=fail \
|
||||
-p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
|
||||
-E "$INTEROP_FILTER"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [minio-interop]
|
||||
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -45,13 +45,6 @@
|
||||
# docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
|
||||
# has fewer cores and reintroduces fleet-state risk for no reliability gain.
|
||||
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||
# reading this file, which has already misled at least one audit — hence this
|
||||
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||
#
|
||||
name: mint
|
||||
|
||||
on:
|
||||
@@ -70,13 +63,13 @@ on:
|
||||
- core
|
||||
- full
|
||||
mint-image:
|
||||
description: "Mint image reference"
|
||||
description: "Mint image reference (empty = pinned default)"
|
||||
required: false
|
||||
default: "minio/mint:edge"
|
||||
default: ""
|
||||
schedule:
|
||||
# Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to
|
||||
# 3h) has finished, so the two never contend for the same runner pool.
|
||||
- cron: "0 6 * * 0"
|
||||
- cron: "41 6 * * 0"
|
||||
|
||||
env:
|
||||
S3_ACCESS_KEY: rustfsadmin-ci
|
||||
|
||||
@@ -16,7 +16,7 @@ name: Nightly GNU Build
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * *"
|
||||
- cron: "7 0 * * *"
|
||||
timezone: "Asia/Shanghai"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -194,3 +194,23 @@ jobs:
|
||||
|
||||
- name: Run HA leader failover live checks (three-node Raft cluster in Docker)
|
||||
run: bash scripts/test/vault_ha_kms_live.sh
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [build, kms-vault-lane, kms-vault-ha-failover]
|
||||
if: >-
|
||||
always() && github.event_name == 'schedule' &&
|
||||
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -22,22 +22,15 @@
|
||||
# correctness cost (e.g. the #4221 fsync durability fix) is recorded, not
|
||||
# blocked (rustfs/backlog#935 correction 1).
|
||||
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||
# reading this file, which has already misled at least one audit — hence this
|
||||
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||
#
|
||||
name: Performance A/B
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 6 * * *" # 06:00 UTC nightly, against main
|
||||
- cron: "31 6 * * *" # 06:31 UTC nightly, against main
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
duration:
|
||||
description: "warp duration per round (short by default to fit the double-build budget)"
|
||||
description: "warp duration per round"
|
||||
required: false
|
||||
default: "12s"
|
||||
type: string
|
||||
@@ -46,12 +39,8 @@ on:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
push:
|
||||
# Every main commit pre-builds and caches its release binary (perf-3) so the
|
||||
# nightly A/B restores a ready baseline instead of paying the double build.
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
|
||||
env:
|
||||
@@ -59,83 +48,19 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# perf-3: on every push to main, build the release binary once and cache it
|
||||
# keyed by commit SHA (rustfs-baseline-<sha>). The warp-ab measurements
|
||||
# restore this instead of paying the ~32min-per-side source
|
||||
# build. That double build is what pushed the expanded 24-cell nightly past its
|
||||
# ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental
|
||||
# builds off the shared cargo cache keep each push cheap, and building on the
|
||||
# same sm-standard-2 runner the A/B measures on guarantees the cached binary is
|
||||
# ABI-identical. Do NOT source this from build.yml's per-merge artifact: those
|
||||
# are cancelled ~7/8 of the time and are not a reliable baseline.
|
||||
build-baseline-cache:
|
||||
name: Build + cache baseline binary
|
||||
if: github.event_name == 'push'
|
||||
runs-on: sm-standard-2
|
||||
# Latest-wins: consumers only ever restore the binary for the *current*
|
||||
# origin/main tip, so when pushes land faster than the ~65min build, a
|
||||
# superseded build's output is dead weight — cancel it instead of stacking
|
||||
# hour-long jobs on the shared runner pool. A skipped intermediate SHA at
|
||||
# most costs one same-commit self-heal in the A/B job.
|
||||
concurrency:
|
||||
group: perf-baseline-build-main
|
||||
cancel-in-progress: true
|
||||
# #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a
|
||||
# single release build past 60min on this runner — every cache build on
|
||||
# 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution
|
||||
# time of 1h0m0s") and the cache never populated. The measured binary must
|
||||
# keep the production profile, so the budget absorbs the build instead.
|
||||
timeout-minutes: 100
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }}
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build release rustfs
|
||||
run: cargo build --release --bin rustfs
|
||||
|
||||
- name: Stage binary for cache
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p baseline-bin
|
||||
cp target/release/rustfs baseline-bin/rustfs
|
||||
|
||||
- name: Cache baseline binary by SHA
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ github.sha }}
|
||||
|
||||
warp-ab:
|
||||
name: Warp A/B budget gate
|
||||
# Always run on schedule / manual dispatch. Never on push — that event only
|
||||
# feeds build-baseline-cache above.
|
||||
if: >-
|
||||
github.event_name == 'schedule' ||
|
||||
github.event_name == 'workflow_dispatch'
|
||||
runs-on: sm-standard-2
|
||||
# With perf-3's cached baseline binary the common (cache-hit) nightly is
|
||||
# measurement-only and finishes well under 50min. This ceiling stays
|
||||
# generous only to absorb the same-commit cache-miss self-heal (~65min
|
||||
# single build with the post-#4806 LTO profile + measurement). A timeout
|
||||
# surfaces via the alert-on-failure job (it fires on cancelled/timed-out,
|
||||
# not just failure). perf-6 recalibrates the budget once the noise study
|
||||
# lands.
|
||||
timeout-minutes: 120
|
||||
# A normal nightly restores the last successful binary and builds only the
|
||||
# candidate; daily access keeps that cache warm. A cache miss may build both
|
||||
# and needs room for the A/B run plus artifact and cache publication.
|
||||
timeout-minutes: 180
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0 # baseline is built from origin/main
|
||||
fetch-depth: 0 # baseline may be an earlier successful scheduled head
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
@@ -163,24 +88,55 @@ jobs:
|
||||
fi
|
||||
echo "allow_regression=$allow" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# perf-3: resolve the commits so the cache can be keyed by SHA. The
|
||||
# baseline is origin/main; the candidate is the checked-out ref. On the
|
||||
# nightly (checkout == main) they are the same commit, so one cached binary
|
||||
# serves both phases and the run does zero source builds.
|
||||
# A failed regression run must keep comparing against the last known-good
|
||||
# scheduled head. Otherwise the next nightly would absorb the regression
|
||||
# into its baseline and turn green without a fix.
|
||||
- name: Find last successful scheduled baseline
|
||||
id: scheduled_baseline
|
||||
if: github.event_name == 'schedule'
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const { data } = await github.rest.actions.listWorkflowRuns({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
workflow_id: "performance-ab.yml",
|
||||
event: "schedule",
|
||||
status: "success",
|
||||
per_page: 1,
|
||||
});
|
||||
return data.workflow_runs[0]?.head_sha ?? "";
|
||||
|
||||
# Manual runs compare a selected ref with current main. Scheduled runs
|
||||
# compare current main with the last successful scheduled head. With no
|
||||
# history, the first run measures the candidate against itself and seeds
|
||||
# that head only if the complete rig succeeds.
|
||||
- name: Resolve baseline / candidate commits
|
||||
id: commits
|
||||
env:
|
||||
SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
baseline_sha="$(git rev-parse origin/main)"
|
||||
candidate_sha="$(git rev-parse HEAD)"
|
||||
if [[ "${{ github.event_name }}" == "schedule" ]]; then
|
||||
baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}"
|
||||
if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then
|
||||
echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
baseline_sha="$(git rev-parse origin/main)"
|
||||
fi
|
||||
git cat-file -e "${baseline_sha}^{commit}"
|
||||
echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline commit: $baseline_sha"
|
||||
echo "candidate commit: $candidate_sha"
|
||||
|
||||
# Exact-key restore of the baseline binary built by build-baseline-cache
|
||||
# when origin/main last landed. A miss (binary evicted or not built yet)
|
||||
# leaves cache-hit unset and the rig falls back to a source build.
|
||||
# Exact-key restore of the candidate binary saved by its successful
|
||||
# scheduled run. A miss leaves cache-hit unset and falls back to a source
|
||||
# build of that known-good head.
|
||||
- name: Restore cached baseline binary
|
||||
id: baseline_cache
|
||||
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
@@ -270,11 +226,11 @@ jobs:
|
||||
elif [[ "$selfheal_built" == "true" ]]; then
|
||||
base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)"
|
||||
else
|
||||
base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)"
|
||||
base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)"
|
||||
fi
|
||||
if [[ "$candidate_sha" == "$baseline_sha" ]]; then
|
||||
# Nightly on main: the candidate is the same commit as the baseline,
|
||||
# so reuse the one binary for both phases and skip all builds.
|
||||
# No commits landed since the last successful baseline, so reuse
|
||||
# the one binary for both phases and measure only rig drift.
|
||||
args+=(--candidate-bin "$base_bin")
|
||||
cand_src="same binary as baseline (same commit)"
|
||||
elif [[ "$candidate_built" == "true" ]]; then
|
||||
@@ -362,6 +318,23 @@ jobs:
|
||||
fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Stage successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cp candidate-bin/rustfs baseline-bin/rustfs
|
||||
|
||||
- name: Cache successful candidate baseline
|
||||
if: >-
|
||||
steps.ab.outputs.status == '0' &&
|
||||
steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha
|
||||
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
|
||||
with:
|
||||
path: baseline-bin/rustfs
|
||||
key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }}
|
||||
|
||||
# Scheduled failure alerting is handled by the alert-on-failure job below
|
||||
# (perf-2 consuming ci-8's schedule-failure-issue composite action).
|
||||
|
||||
|
||||
@@ -30,7 +30,7 @@ name: Runner Hygiene
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
|
||||
- cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Scheduled Validation Freshness
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "47 23 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: scheduled-validation-freshness
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
check-freshness:
|
||||
name: Check scheduled validation freshness
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check latest scheduled runs
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
python3 scripts/check_scheduled_validation_freshness.py \
|
||||
--report "${RUNNER_TEMP}/scheduled-validation-freshness.md"
|
||||
status=$?
|
||||
cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
exit "${status}"
|
||||
- name: Open or update freshness issue
|
||||
if: failure()
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
details-file: ${{ runner.temp }}/scheduled-validation-freshness.md
|
||||
@@ -0,0 +1,63 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Scheduled Validation Watchdog
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows:
|
||||
- "Security Audit"
|
||||
- "Build and Release"
|
||||
- "Continuous Integration"
|
||||
- "coverage"
|
||||
- "e2e-nightly"
|
||||
- "e2e-s3tests"
|
||||
- "Fuzz"
|
||||
- "mint"
|
||||
- "minio-interop"
|
||||
- "Nightly GNU Build"
|
||||
- "Performance A/B"
|
||||
- "Runner Hygiene"
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
alert-on-incomplete-run:
|
||||
name: Alert on incomplete scheduled run
|
||||
if: >-
|
||||
github.event.workflow_run.event == 'schedule' &&
|
||||
github.event.workflow_run.conclusion != 'success' &&
|
||||
github.event.workflow_run.conclusion != 'failure'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update incomplete-run issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
workflow-name: ${{ github.event.workflow_run.name }}
|
||||
source-run-id: ${{ github.event.workflow_run.id }}
|
||||
source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
|
||||
source-event: ${{ github.event.workflow_run.event }}
|
||||
source-ref-name: ${{ github.event.workflow_run.head_branch }}
|
||||
source-sha: ${{ github.event.workflow_run.head_sha }}
|
||||
@@ -901,6 +901,10 @@ pub struct Metrics {
|
||||
scanner_cycle_max_duration_millis: AtomicU64,
|
||||
scanner_cycle_max_objects: AtomicU64,
|
||||
scanner_cycle_max_directories: AtomicU64,
|
||||
scanner_cycle_timeout_total: AtomicU64,
|
||||
scanner_cycle_recovery_required_total: AtomicU64,
|
||||
scanner_cycle_last_progress_age_seconds: AtomicU64,
|
||||
scanner_leader_lease_without_progress: AtomicBool,
|
||||
scanner_bitrot_cycle_enabled: AtomicBool,
|
||||
scanner_bitrot_cycle_millis: AtomicU64,
|
||||
scanner_checkpoint: Mutex<Option<ScannerCheckpointReport>>,
|
||||
@@ -1370,6 +1374,14 @@ pub struct ScannerMetricsReport {
|
||||
#[serde(default)]
|
||||
pub cycle_max_directories: u64,
|
||||
#[serde(default)]
|
||||
pub cycle_timeout_total: u64,
|
||||
#[serde(default)]
|
||||
pub cycle_recovery_required_total: u64,
|
||||
#[serde(default)]
|
||||
pub cycle_last_progress_age: u64,
|
||||
#[serde(default)]
|
||||
pub leader_lease_without_progress: bool,
|
||||
#[serde(default)]
|
||||
pub bitrot_cycle_enabled: bool,
|
||||
#[serde(default)]
|
||||
pub bitrot_cycle_seconds: f64,
|
||||
@@ -1430,6 +1442,9 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total
|
||||
const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total";
|
||||
const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds";
|
||||
const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds";
|
||||
const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total";
|
||||
const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age";
|
||||
const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress";
|
||||
|
||||
fn scan_cycle_result_label(result: u8) -> &'static str {
|
||||
match result {
|
||||
@@ -1913,6 +1928,10 @@ impl Metrics {
|
||||
scanner_cycle_max_duration_millis: AtomicU64::new(0),
|
||||
scanner_cycle_max_objects: AtomicU64::new(0),
|
||||
scanner_cycle_max_directories: AtomicU64::new(0),
|
||||
scanner_cycle_timeout_total: AtomicU64::new(0),
|
||||
scanner_cycle_recovery_required_total: AtomicU64::new(0),
|
||||
scanner_cycle_last_progress_age_seconds: AtomicU64::new(0),
|
||||
scanner_leader_lease_without_progress: AtomicBool::new(false),
|
||||
scanner_bitrot_cycle_enabled: AtomicBool::new(false),
|
||||
scanner_bitrot_cycle_millis: AtomicU64::new(0),
|
||||
scanner_checkpoint: Mutex::new(None),
|
||||
@@ -2412,12 +2431,29 @@ impl Metrics {
|
||||
.store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed);
|
||||
self.scanner_cycle_max_directories
|
||||
.store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed);
|
||||
self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed);
|
||||
self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed);
|
||||
metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0);
|
||||
metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0);
|
||||
self.scanner_bitrot_cycle_enabled
|
||||
.store(bitrot_cycle.is_some(), Ordering::Relaxed);
|
||||
self.scanner_bitrot_cycle_millis
|
||||
.store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) {
|
||||
self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed);
|
||||
if recovery_required {
|
||||
self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.scanner_cycle_last_progress_age_seconds
|
||||
.store(progress_age.as_secs(), Ordering::Relaxed);
|
||||
self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed);
|
||||
metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1);
|
||||
metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64());
|
||||
metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0);
|
||||
}
|
||||
|
||||
pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option<usize>, queued: Option<usize>, active: Option<usize>) {
|
||||
if let Some(concurrency_limit) = concurrency_limit {
|
||||
self.scanner_set_scan_concurrency_limit
|
||||
@@ -3265,6 +3301,10 @@ impl Metrics {
|
||||
m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed);
|
||||
m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed);
|
||||
m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed);
|
||||
m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed);
|
||||
m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed);
|
||||
m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed);
|
||||
m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed);
|
||||
m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.scan_checkpoint = match self.scanner_checkpoint.lock() {
|
||||
@@ -4926,4 +4966,20 @@ mod tests {
|
||||
assert!(!report.bitrot_cycle_enabled);
|
||||
assert_eq!(report.bitrot_cycle_seconds, 0.0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17));
|
||||
let timed_out = metrics.report().await;
|
||||
assert_eq!(timed_out.cycle_timeout_total, 1);
|
||||
assert_eq!(timed_out.cycle_last_progress_age, 17);
|
||||
assert!(timed_out.leader_lease_without_progress);
|
||||
|
||||
metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None);
|
||||
let current = metrics.report().await;
|
||||
assert_eq!(current.cycle_timeout_total, 1);
|
||||
assert_eq!(current.cycle_last_progress_age, 0);
|
||||
assert!(!current.leader_lease_without_progress);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,6 +84,12 @@ Current guidance:
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
|
||||
|
||||
Scanner cycle budget controls:
|
||||
|
||||
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
|
||||
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
|
||||
- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
|
||||
|
||||
## Mmap read environment aliases
|
||||
|
||||
- `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical)
|
||||
|
||||
@@ -143,9 +143,12 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
|
||||
/// Default scanner speed preset.
|
||||
pub const DEFAULT_SCANNER_SPEED: &str = "default";
|
||||
|
||||
/// Default scanner cycle runtime budget.
|
||||
/// `0` keeps the existing unbounded per-cycle behavior.
|
||||
pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0;
|
||||
/// Default scanner cycle runtime budget when no override is configured.
|
||||
///
|
||||
/// An explicit `0` remains the compatibility escape hatch for an unbounded
|
||||
/// cycle. Keeping the unset default finite prevents a stalled scanner I/O
|
||||
/// operation from holding the leader lease forever.
|
||||
pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60;
|
||||
|
||||
/// Default scanner per-cycle object budget.
|
||||
/// `0` keeps the existing unbounded per-cycle behavior.
|
||||
|
||||
@@ -585,9 +585,12 @@ impl VersionsHistogram {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication statistics for a single target
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicationStats {
|
||||
/// Replication statistics for a single target.
|
||||
///
|
||||
/// Renamed from `ReplicationStats`; serde field names are preserved
|
||||
/// byte-identically to maintain wire compatibility with existing snapshots.
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ReplicationTargetUsage {
|
||||
pub pending_size: u64,
|
||||
pub replicated_size: u64,
|
||||
pub failed_size: u64,
|
||||
@@ -600,7 +603,7 @@ pub struct ReplicationStats {
|
||||
pub replicated_count: u64,
|
||||
}
|
||||
|
||||
impl ReplicationStats {
|
||||
impl ReplicationTargetUsage {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
let Self {
|
||||
pending_size,
|
||||
@@ -636,7 +639,7 @@ impl ReplicationStats {
|
||||
/// Replication statistics for all targets
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicationAllStats {
|
||||
pub targets: HashMap<String, ReplicationStats>,
|
||||
pub targets: HashMap<String, ReplicationTargetUsage>,
|
||||
pub replica_size: u64,
|
||||
pub replica_count: u64,
|
||||
}
|
||||
@@ -649,7 +652,7 @@ impl ReplicationAllStats {
|
||||
targets,
|
||||
} = self;
|
||||
|
||||
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty)
|
||||
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationTargetUsage::is_empty)
|
||||
}
|
||||
|
||||
#[deprecated(note = "use is_empty instead")]
|
||||
@@ -2466,7 +2469,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn replication_stats_empty_checks_every_field() {
|
||||
type SetField = fn(&mut ReplicationStats);
|
||||
type SetField = fn(&mut ReplicationTargetUsage);
|
||||
|
||||
let cases: [(&str, SetField); 10] = [
|
||||
("pending_size", |stats| stats.pending_size = 1),
|
||||
@@ -2481,9 +2484,9 @@ mod tests {
|
||||
("replicated_count", |stats| stats.replicated_count = 1),
|
||||
];
|
||||
|
||||
assert!(ReplicationStats::default().is_empty());
|
||||
assert!(ReplicationTargetUsage::default().is_empty());
|
||||
for (field, set_nonzero) in cases {
|
||||
let mut stats = ReplicationStats::default();
|
||||
let mut stats = ReplicationTargetUsage::default();
|
||||
set_nonzero(&mut stats);
|
||||
assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
|
||||
}
|
||||
@@ -2514,17 +2517,17 @@ mod tests {
|
||||
}
|
||||
|
||||
let empty_targets = ReplicationAllStats {
|
||||
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]),
|
||||
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationTargetUsage::default())]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
|
||||
|
||||
let stats = ReplicationAllStats {
|
||||
targets: HashMap::from([
|
||||
("arn:test:empty".to_string(), ReplicationStats::default()),
|
||||
("arn:test:empty".to_string(), ReplicationTargetUsage::default()),
|
||||
(
|
||||
"arn:test:non-empty".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2565,7 +2568,7 @@ mod tests {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:pending".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2714,7 +2717,7 @@ mod tests {
|
||||
targets: HashMap::from([
|
||||
(
|
||||
"arn:self-only".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
pending_size: 7,
|
||||
pending_count: 1,
|
||||
..Default::default()
|
||||
@@ -2722,7 +2725,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
"arn:shared".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
failed_size: 3,
|
||||
failed_count: 1,
|
||||
missed_threshold_size: 2,
|
||||
@@ -2741,7 +2744,7 @@ mod tests {
|
||||
targets: HashMap::from([
|
||||
(
|
||||
"arn:shared".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
failed_size: 5,
|
||||
failed_count: 2,
|
||||
after_threshold_size: 4,
|
||||
@@ -2751,7 +2754,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
"arn:other-only".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
replicated_size: 11,
|
||||
replicated_count: 3,
|
||||
..Default::default()
|
||||
@@ -2993,7 +2996,9 @@ mod tests {
|
||||
fn replication_target_deserialization_preserves_large_historical_maps() {
|
||||
let mut stats = ReplicationAllStats::default();
|
||||
for index in 0..=1024 {
|
||||
stats.targets.insert(format!("target-{index}"), ReplicationStats::default());
|
||||
stats
|
||||
.targets
|
||||
.insert(format!("target-{index}"), ReplicationTargetUsage::default());
|
||||
}
|
||||
let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode");
|
||||
let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded)
|
||||
@@ -3002,6 +3007,47 @@ mod tests {
|
||||
assert_eq!(decoded.targets.len(), stats.targets.len());
|
||||
}
|
||||
|
||||
/// Round-trip test: encoding a [`ReplicationTargetUsage`] and decoding it back
|
||||
/// must produce the exact same value. This guards against accidental serde
|
||||
/// field-name drift during the `ReplicationStats` -> `ReplicationTargetUsage`
|
||||
/// rename. Wire-level field names are the serialized Rust field identifiers,
|
||||
/// which must remain byte-identical.
|
||||
#[test]
|
||||
fn replication_target_usage_rmp_round_trip() {
|
||||
let original = ReplicationTargetUsage {
|
||||
pending_size: 100,
|
||||
replicated_size: 2_000,
|
||||
failed_size: 50,
|
||||
failed_count: 3,
|
||||
pending_count: 7,
|
||||
missed_threshold_size: 11,
|
||||
after_threshold_size: 22,
|
||||
missed_threshold_count: 1,
|
||||
after_threshold_count: 2,
|
||||
replicated_count: 99,
|
||||
};
|
||||
|
||||
let buf = rmp_serde::to_vec_named(&original).expect("encode ReplicationTargetUsage to msgpack");
|
||||
let decoded: ReplicationTargetUsage = rmp_serde::from_slice(&buf).expect("decode ReplicationTargetUsage from msgpack");
|
||||
assert_eq!(original, decoded, "round-trip through rmp must preserve every field");
|
||||
|
||||
// Also verify that encoding as an unnamed sequence and then decoding
|
||||
// with named fields produces the correct mapping (this catches reordering).
|
||||
let named_buf = rmp_serde::to_vec_named(&original).expect("re-encode for field-name pinning");
|
||||
// Spot-check that known field names appear in the named encoding.
|
||||
let named_str = String::from_utf8_lossy(&named_buf);
|
||||
assert!(named_str.contains("pending_size"), "field 'pending_size' must survive the rename");
|
||||
assert!(named_str.contains("replicated_size"), "field 'replicated_size' must survive the rename");
|
||||
assert!(
|
||||
named_str.contains("missed_threshold_size"),
|
||||
"field 'missed_threshold_size' must survive the rename"
|
||||
);
|
||||
assert!(
|
||||
named_str.contains("after_threshold_count"),
|
||||
"field 'after_threshold_count' must survive the rename"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checked_merge_rejects_noncanonical_histograms_without_mutation() {
|
||||
let mut entry = DataUsageEntry {
|
||||
|
||||
@@ -380,10 +380,24 @@ mod tests {
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
assert!(
|
||||
!status_body.contains("MissingContentLength"),
|
||||
"background heal status should not fail without an explicit Content-Length: {status_body}"
|
||||
let mut recovered = serde_json::Value::Null;
|
||||
for _ in 0..60 {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
assert!(
|
||||
!status_body.contains("MissingContentLength"),
|
||||
"background heal status should not fail without an explicit Content-Length: {status_body}"
|
||||
);
|
||||
recovered = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if recovered["clusterStatusComplete"] == serde_json::Value::Bool(true) {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
assert_eq!(
|
||||
recovered["clusterStatusComplete"],
|
||||
serde_json::Value::Bool(true),
|
||||
"cluster heal status should recover before root heal starts: {recovered}"
|
||||
);
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
|
||||
@@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box<dyn std::erro
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -159,7 +159,7 @@ async fn test_bucket_default_sse_kms_put_object() -> Result<(), Box<dyn std::err
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -278,7 +278,7 @@ async fn test_bucket_default_sse_kms_multipart_crc32() -> Result<(), Box<dyn std
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -475,7 +475,7 @@ async fn test_explicit_encryption_overrides_bucket_default() -> Result<(), Box<d
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -570,7 +570,7 @@ async fn test_sse_kms_without_key_id_populates_default() -> Result<(), Box<dyn s
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -189,34 +189,121 @@ pub async fn wait_for_kms_ready(
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let total_deadline = Duration::from_secs(5);
|
||||
wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await
|
||||
}
|
||||
|
||||
async fn wait_for_kms_ready_with_timeout(
|
||||
base_url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
total_deadline: Duration,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let start = tokio::time::Instant::now();
|
||||
let deadline = start + total_deadline;
|
||||
let mut backoff = Duration::from_millis(200);
|
||||
let max_backoff = Duration::from_secs(1);
|
||||
let mut first_attempt = true;
|
||||
|
||||
loop {
|
||||
if !first_attempt {
|
||||
if start.elapsed() >= total_deadline {
|
||||
return Err("KMS failed to become ready within 5 seconds".into());
|
||||
}
|
||||
sleep(backoff).await;
|
||||
backoff = (backoff * 2).min(max_backoff);
|
||||
}
|
||||
first_attempt = false;
|
||||
|
||||
match get_kms_status(base_url, access_key, secret_key).await {
|
||||
Ok(status) => {
|
||||
info!("KMS is ready (status: {})", status);
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
if start.elapsed() >= total_deadline {
|
||||
return Err(format!("KMS did not become ready within 5 s: last error: {e}").into());
|
||||
match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await {
|
||||
Ok(Ok(status)) => {
|
||||
let backend_status = serde_json::from_str::<serde_json::Value>(&status)
|
||||
.ok()
|
||||
.and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned));
|
||||
if backend_status.as_deref() == Some("healthy") {
|
||||
info!("KMS is ready (status: {})", status);
|
||||
return Ok(());
|
||||
}
|
||||
warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…");
|
||||
warn!(
|
||||
backend_status = backend_status.as_deref().unwrap_or("missing"),
|
||||
elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
"KMS not ready yet, retrying…"
|
||||
);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…");
|
||||
}
|
||||
Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()),
|
||||
}
|
||||
|
||||
let now = tokio::time::Instant::now();
|
||||
if now >= deadline {
|
||||
return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into());
|
||||
}
|
||||
sleep((now + backoff).min(deadline) - now).await;
|
||||
backoff = (backoff * 2).min(max_backoff);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod readiness_tests {
|
||||
use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[tokio::test]
|
||||
async fn kms_readiness_retries_http_success_until_backend_is_healthy() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
|
||||
let address = listener.local_addr().expect("read readiness test server address");
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let server_requests = Arc::clone(&requests);
|
||||
let server = tokio::spawn(async move {
|
||||
for backend_status in ["error", "healthy"] {
|
||||
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
|
||||
let mut request = Vec::new();
|
||||
let mut chunk = [0_u8; 1024];
|
||||
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let read = socket.read(&mut chunk).await.expect("read readiness request");
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
request.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
server_requests.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
let body = format!(r#"{{"backend_status":"{backend_status}"}}"#);
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
socket.write_all(response.as_bytes()).await.expect("write readiness response");
|
||||
}
|
||||
});
|
||||
|
||||
wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key")
|
||||
.await
|
||||
.expect("KMS should become ready after the healthy response");
|
||||
|
||||
let observed_requests = requests.load(Ordering::SeqCst);
|
||||
server.abort();
|
||||
assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kms_readiness_deadline_covers_a_stalled_status_request() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server");
|
||||
let address = listener.local_addr().expect("read readiness test server address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.expect("accept readiness request");
|
||||
let mut request = [0_u8; 1024];
|
||||
let _ = socket.read(&mut request).await.expect("read readiness request");
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
|
||||
let result = tokio::time::timeout(
|
||||
Duration::from_secs(1),
|
||||
wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)),
|
||||
)
|
||||
.await
|
||||
.expect("readiness helper must enforce its own deadline");
|
||||
|
||||
server.abort();
|
||||
assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() {
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
// Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service
|
||||
@@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() {
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
// Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below
|
||||
@@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
let bucket = "copy-object-self-copy-bucket-default-sse-test";
|
||||
|
||||
@@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() {
|
||||
)
|
||||
.await
|
||||
.expect("failed to start RustFS with local KMS");
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
|
||||
let client = kms_env.base_env.create_s3_client();
|
||||
let bucket = "copy-object-version-restore-sse-test";
|
||||
|
||||
@@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box<dyn s
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -147,7 +147,7 @@ async fn test_head_reports_managed_metadata_for_sse_kms_and_copy() -> Result<(),
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box<dyn std
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -24,7 +24,6 @@ use super::common::{
|
||||
test_sse_kms_encryption, test_sse_s3_encryption,
|
||||
};
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::info;
|
||||
|
||||
/// Comprehensive test: Full KMS workflow with all encryption types
|
||||
@@ -35,7 +34,7 @@ async fn test_comprehensive_kms_full_workflow() -> Result<(), Box<dyn std::error
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -103,7 +102,7 @@ async fn test_comprehensive_stress_test() -> Result<(), Box<dyn std::error::Erro
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -137,7 +136,7 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box<dyn std::error::Er
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -208,7 +207,7 @@ async fn test_comprehensive_concurrent_operations() -> Result<(), Box<dyn std::e
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -253,7 +252,7 @@ async fn test_comprehensive_performance_benchmark() -> Result<(), Box<dyn std::e
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -44,7 +44,7 @@ async fn test_kms_zero_byte_file_encryption() -> Result<(), Box<dyn std::error::
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -117,7 +117,7 @@ async fn test_kms_single_byte_file_encryption() -> Result<(), Box<dyn std::error
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -209,7 +209,7 @@ async fn test_kms_multipart_boundary_conditions() -> Result<(), Box<dyn std::err
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -284,7 +284,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box<dyn std::error::Erro
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -371,7 +371,7 @@ async fn test_kms_concurrent_encryption() -> Result<(), Box<dyn std::error::Erro
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = Arc::new(kms_env.base_env.create_s3_client());
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -478,7 +478,7 @@ async fn test_kms_key_validation_security() -> Result<(), Box<dyn std::error::Er
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -37,7 +37,7 @@ async fn test_kms_key_directory_unavailable() -> Result<(), Box<dyn std::error::
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -127,7 +127,7 @@ async fn test_kms_corrupted_key_files() -> Result<(), Box<dyn std::error::Error
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -218,7 +218,7 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box<dyn std::err
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -401,7 +401,7 @@ async fn test_kms_resource_constraints() -> Result<(), Box<dyn std::error::Error
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -46,7 +46,7 @@ async fn test_local_kms_end_to_end() -> Result<(), Box<dyn std::error::Error + S
|
||||
.expect("Failed to start RustFS with Local KMS");
|
||||
|
||||
// Wait a moment for RustFS to fully start up and initialize KMS
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
|
||||
|
||||
@@ -127,7 +127,7 @@ async fn test_local_kms_key_isolation() {
|
||||
.expect("Failed to start RustFS with Local KMS");
|
||||
|
||||
// Wait a moment for RustFS to fully start up and initialize KMS
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
|
||||
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
|
||||
|
||||
@@ -227,7 +227,7 @@ async fn test_local_kms_large_file() {
|
||||
.expect("Failed to start RustFS with Local KMS");
|
||||
|
||||
// Wait a moment for RustFS to fully start up and initialize KMS
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
|
||||
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
|
||||
|
||||
@@ -309,7 +309,7 @@ async fn test_local_kms_multipart_upload() {
|
||||
.expect("Failed to start RustFS with Local KMS");
|
||||
|
||||
// Wait for KMS initialization
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await.expect("KMS ready");
|
||||
|
||||
info!("RustFS started with KMS auto-configuration, default_key_id: {}", default_key_id);
|
||||
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
//! multipart upload behaviour.
|
||||
|
||||
use crate::common::{TEST_BUCKET, init_logging};
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::{error, info};
|
||||
|
||||
use super::common::{
|
||||
@@ -45,8 +44,8 @@ impl VaultKmsTestContext {
|
||||
|
||||
start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
|
||||
|
||||
// Allow Vault to finish initialising token auth and transit engine.
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
// Wait for KMS to finish initialising.
|
||||
super::common::wait_for_kms_ready(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
|
||||
|
||||
Ok(Self { env })
|
||||
}
|
||||
|
||||
@@ -39,9 +39,6 @@ mod kms_edge_cases_test;
|
||||
#[cfg(test)]
|
||||
mod kms_fault_recovery_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_runner;
|
||||
|
||||
#[cfg(test)]
|
||||
mod bucket_default_encryption_test;
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ async fn test_step1_basic_single_file_encryption() -> Result<(), Box<dyn std::er
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -89,7 +89,7 @@ async fn test_step2_basic_multipart_upload_without_encryption() -> Result<(), Bo
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box<dyn std::er
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -310,7 +310,7 @@ async fn test_step4_large_multipart_upload_with_encryption() -> Result<(), Box<d
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
@@ -435,7 +435,7 @@ async fn test_step5_all_encryption_types_multipart() -> Result<(), Box<dyn std::
|
||||
|
||||
let mut kms_env = LocalKMSTestEnvironment::new().await?;
|
||||
let _default_key_id = kms_env.start_rustfs_for_local_kms().await?;
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
|
||||
kms_env.wait_for_kms_ready().await?;
|
||||
|
||||
let s3_client = kms_env.base_env.create_s3_client();
|
||||
kms_env.base_env.create_test_bucket(TEST_BUCKET).await?;
|
||||
|
||||
@@ -1,499 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
#![allow(dead_code)]
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Unified KMS test suite runner
|
||||
//!
|
||||
//! This module provides a unified interface for running KMS tests with categorization,
|
||||
//! filtering, and comprehensive reporting capabilities.
|
||||
|
||||
use crate::common::init_logging;
|
||||
use std::time::Instant;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// Test category for organization and filtering
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub enum TestCategory {
|
||||
CoreFunctionality,
|
||||
MultipartEncryption,
|
||||
EdgeCases,
|
||||
FaultRecovery,
|
||||
Comprehensive,
|
||||
Performance,
|
||||
}
|
||||
|
||||
impl TestCategory {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
TestCategory::CoreFunctionality => "core-functionality",
|
||||
TestCategory::MultipartEncryption => "multipart-encryption",
|
||||
TestCategory::EdgeCases => "edge-cases",
|
||||
TestCategory::FaultRecovery => "fault-recovery",
|
||||
TestCategory::Comprehensive => "comprehensive",
|
||||
TestCategory::Performance => "performance",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Test definition with metadata
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestDefinition {
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
pub category: TestCategory,
|
||||
pub estimated_duration: Duration,
|
||||
pub is_critical: bool,
|
||||
}
|
||||
|
||||
impl TestDefinition {
|
||||
pub fn new(
|
||||
name: impl Into<String>,
|
||||
description: impl Into<String>,
|
||||
category: TestCategory,
|
||||
estimated_duration: Duration,
|
||||
is_critical: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
name: name.into(),
|
||||
description: description.into(),
|
||||
category,
|
||||
estimated_duration,
|
||||
is_critical,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Test execution result
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestResult {
|
||||
pub test_name: String,
|
||||
pub category: TestCategory,
|
||||
pub success: bool,
|
||||
pub duration: Duration,
|
||||
pub error_message: Option<String>,
|
||||
}
|
||||
|
||||
impl TestResult {
|
||||
pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self {
|
||||
Self {
|
||||
test_name,
|
||||
category,
|
||||
success: true,
|
||||
duration,
|
||||
error_message: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self {
|
||||
Self {
|
||||
test_name,
|
||||
category,
|
||||
success: false,
|
||||
duration,
|
||||
error_message: Some(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Comprehensive test suite configuration
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestSuiteConfig {
|
||||
pub categories: Vec<TestCategory>,
|
||||
pub include_critical_only: bool,
|
||||
pub max_duration: Option<Duration>,
|
||||
pub parallel_execution: bool,
|
||||
}
|
||||
|
||||
impl Default for TestSuiteConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
categories: vec![
|
||||
TestCategory::CoreFunctionality,
|
||||
TestCategory::MultipartEncryption,
|
||||
TestCategory::EdgeCases,
|
||||
TestCategory::FaultRecovery,
|
||||
TestCategory::Comprehensive,
|
||||
],
|
||||
include_critical_only: false,
|
||||
max_duration: None,
|
||||
parallel_execution: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Unified KMS test suite runner
|
||||
pub struct KMSTestSuite {
|
||||
tests: Vec<TestDefinition>,
|
||||
config: TestSuiteConfig,
|
||||
}
|
||||
|
||||
impl KMSTestSuite {
|
||||
/// Create a new test suite with default configuration
|
||||
pub fn new() -> Self {
|
||||
let tests = vec![
|
||||
// Core Functionality Tests
|
||||
TestDefinition::new(
|
||||
"test_local_kms_end_to_end",
|
||||
"End-to-end KMS test with all encryption types",
|
||||
TestCategory::CoreFunctionality,
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_local_kms_key_isolation",
|
||||
"Test KMS key isolation and security",
|
||||
TestCategory::CoreFunctionality,
|
||||
Duration::from_secs(45),
|
||||
true,
|
||||
),
|
||||
// Multipart Encryption Tests
|
||||
TestDefinition::new(
|
||||
"test_local_kms_multipart_upload",
|
||||
"Test large file multipart upload with encryption",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(120),
|
||||
true,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step1_basic_single_file_encryption",
|
||||
"Basic single file encryption test",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(30),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step2_basic_multipart_upload_without_encryption",
|
||||
"Basic multipart upload without encryption",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(45),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step3_multipart_upload_with_sse_s3",
|
||||
"Multipart upload with SSE-S3 encryption",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step4_large_multipart_upload_with_encryption",
|
||||
"Large file multipart upload with encryption",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(90),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_step5_all_encryption_types_multipart",
|
||||
"All encryption types multipart test",
|
||||
TestCategory::MultipartEncryption,
|
||||
Duration::from_secs(120),
|
||||
true,
|
||||
),
|
||||
// Edge Cases Tests
|
||||
TestDefinition::new(
|
||||
"test_kms_zero_byte_file_encryption",
|
||||
"Test encryption of zero-byte files",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(20),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_single_byte_file_encryption",
|
||||
"Test encryption of single-byte files",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(20),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_multipart_boundary_conditions",
|
||||
"Test multipart upload boundary conditions",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(45),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_invalid_key_scenarios",
|
||||
"Test invalid key scenarios",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(30),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_concurrent_encryption",
|
||||
"Test concurrent encryption operations",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(60),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_key_validation_security",
|
||||
"Test key validation security",
|
||||
TestCategory::EdgeCases,
|
||||
Duration::from_secs(30),
|
||||
false,
|
||||
),
|
||||
// Fault Recovery Tests
|
||||
TestDefinition::new(
|
||||
"test_kms_key_directory_unavailable",
|
||||
"Test KMS when key directory is unavailable",
|
||||
TestCategory::FaultRecovery,
|
||||
Duration::from_secs(45),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_corrupted_key_files",
|
||||
"Test KMS with corrupted key files",
|
||||
TestCategory::FaultRecovery,
|
||||
Duration::from_secs(30),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_multipart_upload_interruption",
|
||||
"Test multipart upload interruption recovery",
|
||||
TestCategory::FaultRecovery,
|
||||
Duration::from_secs(60),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_kms_resource_constraints",
|
||||
"Test KMS under resource constraints",
|
||||
TestCategory::FaultRecovery,
|
||||
Duration::from_secs(90),
|
||||
false,
|
||||
),
|
||||
// Comprehensive Tests
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_kms_full_workflow",
|
||||
"Full KMS workflow comprehensive test",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(300),
|
||||
true,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_stress_test",
|
||||
"KMS stress test with large datasets",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(400),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_key_isolation",
|
||||
"Comprehensive key isolation test",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(180),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_concurrent_operations",
|
||||
"Comprehensive concurrent operations test",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(240),
|
||||
false,
|
||||
),
|
||||
TestDefinition::new(
|
||||
"test_comprehensive_performance_benchmark",
|
||||
"KMS performance benchmark test",
|
||||
TestCategory::Comprehensive,
|
||||
Duration::from_secs(360),
|
||||
false,
|
||||
),
|
||||
];
|
||||
|
||||
Self {
|
||||
tests,
|
||||
config: TestSuiteConfig::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Configure the test suite
|
||||
pub fn with_config(mut self, config: TestSuiteConfig) -> Self {
|
||||
self.config = config;
|
||||
self
|
||||
}
|
||||
|
||||
/// Filter tests based on category
|
||||
pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> {
|
||||
self.tests.iter().filter(|test| &test.category == category).collect()
|
||||
}
|
||||
|
||||
/// Filter tests based on criticality
|
||||
pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> {
|
||||
self.tests.iter().filter(|test| test.is_critical).collect()
|
||||
}
|
||||
|
||||
/// Get test summary by category
|
||||
pub fn get_category_summary(&self) -> std::collections::HashMap<TestCategory, Vec<&TestDefinition>> {
|
||||
let mut summary = std::collections::HashMap::new();
|
||||
for test in &self.tests {
|
||||
summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test);
|
||||
}
|
||||
summary
|
||||
}
|
||||
|
||||
/// Run the complete test suite
|
||||
pub async fn run_test_suite(&self) -> Vec<TestResult> {
|
||||
init_logging();
|
||||
info!("🚀 Starting unified KMS test suite");
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut results = Vec::new();
|
||||
|
||||
// Filter tests based on configuration
|
||||
let tests_to_run: Vec<&TestDefinition> = self
|
||||
.tests
|
||||
.iter()
|
||||
.filter(|test| self.config.categories.contains(&test.category))
|
||||
.filter(|test| !self.config.include_critical_only || test.is_critical)
|
||||
.collect();
|
||||
|
||||
info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len());
|
||||
for (i, test) in tests_to_run.iter().enumerate() {
|
||||
info!(" {}. {} ({})", i + 1, test.name, test.category.as_str());
|
||||
}
|
||||
|
||||
// Execute tests
|
||||
for (i, test_def) in tests_to_run.iter().enumerate() {
|
||||
info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name);
|
||||
info!(" 📝 Description: {}", test_def.description);
|
||||
info!(" 🏷️ Category: {}", test_def.category.as_str());
|
||||
info!(" ⏱️ Estimated duration: {:?}", test_def.estimated_duration);
|
||||
|
||||
let test_start = Instant::now();
|
||||
let result = self.run_single_test(test_def).await;
|
||||
let test_duration = test_start.elapsed();
|
||||
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64());
|
||||
results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration));
|
||||
}
|
||||
Err(e) => {
|
||||
error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e);
|
||||
results.push(TestResult::failure(
|
||||
test_def.name.clone(),
|
||||
test_def.category.clone(),
|
||||
test_duration,
|
||||
e.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Add delay between tests to avoid resource conflicts
|
||||
if i < tests_to_run.len() - 1 {
|
||||
debug!("⏸️ Waiting two seconds before the next test...");
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
}
|
||||
}
|
||||
|
||||
let total_duration = start_time.elapsed();
|
||||
self.print_test_summary(&results, total_duration);
|
||||
|
||||
results
|
||||
}
|
||||
|
||||
/// Run a single test by dispatching to the appropriate test function
|
||||
async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
// This is a placeholder for test dispatch logic
|
||||
// In a real implementation, this would dispatch to actual test functions
|
||||
warn!("⚠️ Test '{}' is not implemented in the unified runner; skipping", test_def.name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Print comprehensive test summary
|
||||
fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) {
|
||||
info!("📊 KMS test suite summary");
|
||||
info!("⏱️ Total duration: {:.2} seconds", total_duration.as_secs_f64());
|
||||
info!("📈 Total tests: {}", results.len());
|
||||
|
||||
let passed = results.iter().filter(|r| r.success).count();
|
||||
let failed = results.iter().filter(|r| !r.success).count();
|
||||
|
||||
info!("✅ Passed: {}", passed);
|
||||
info!("❌ Failed: {}", failed);
|
||||
info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0);
|
||||
|
||||
// Summary by category
|
||||
let mut category_summary: std::collections::HashMap<TestCategory, (usize, usize)> = std::collections::HashMap::new();
|
||||
for result in results {
|
||||
let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0));
|
||||
*total += 1;
|
||||
if result.success {
|
||||
*passed_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
info!("📊 Category summary:");
|
||||
for (category, (total, passed_count)) in category_summary {
|
||||
info!(
|
||||
" 🏷️ {}: {}/{} ({:.1}%)",
|
||||
category.as_str(),
|
||||
passed_count,
|
||||
total,
|
||||
(passed_count as f64 / total as f64) * 100.0
|
||||
);
|
||||
}
|
||||
|
||||
// List failed tests
|
||||
if failed > 0 {
|
||||
warn!("❌ Failing tests:");
|
||||
for result in results.iter().filter(|r| !r.success) {
|
||||
warn!(" - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Quick test suite for critical tests only
|
||||
#[tokio::test]
|
||||
async fn test_kms_critical_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let config = TestSuiteConfig {
|
||||
categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption],
|
||||
include_critical_only: true,
|
||||
max_duration: Some(Duration::from_secs(600)), // 10 minutes max
|
||||
parallel_execution: false,
|
||||
};
|
||||
|
||||
let suite = KMSTestSuite::new().with_config(config);
|
||||
let results = suite.run_test_suite().await;
|
||||
|
||||
let failed_count = results.iter().filter(|r| !r.success).count();
|
||||
if failed_count > 0 {
|
||||
return Err(format!("Critical test suite failed: {failed_count} tests failed").into());
|
||||
}
|
||||
|
||||
info!("✅ All critical tests passed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Full comprehensive test suite
|
||||
#[tokio::test]
|
||||
async fn test_kms_full_suite() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let suite = KMSTestSuite::new();
|
||||
let results = suite.run_test_suite().await;
|
||||
|
||||
let total_tests = results.len();
|
||||
let failed_count = results.iter().filter(|r| !r.success).count();
|
||||
let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0;
|
||||
|
||||
info!("📊 Full suite success rate: {:.1}%", success_rate);
|
||||
|
||||
// Allow up to 10% failure rate for non-critical tests
|
||||
if success_rate < 90.0 {
|
||||
return Err(format!("Test suite success rate too low: {success_rate:.1}%").into());
|
||||
}
|
||||
|
||||
info!("✅ Full test suite succeeded");
|
||||
Ok(())
|
||||
}
|
||||
@@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE};
|
||||
use rustfs_signer::pre_sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use s3s::Body;
|
||||
@@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
|
||||
.await?;
|
||||
|
||||
let bucket = "object-lambda-e2e-disabled-target";
|
||||
let key = "input.txt";
|
||||
@@ -992,17 +994,24 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box<dyn
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
configure_webhook_target_with_key_values(
|
||||
&env,
|
||||
"transformer",
|
||||
vec![
|
||||
("endpoint", "http://127.0.0.1:9/transform".to_string()),
|
||||
("auth_token", "secret-token".to_string()),
|
||||
("enable", "off".to_string()),
|
||||
],
|
||||
let queue_dir = format!("{}/disabled-target-queue", env.temp_dir);
|
||||
tokio::fs::create_dir_all(&queue_dir).await?;
|
||||
let config_url = format!("{}/rustfs/admin/v3/set-config-kv", env.url);
|
||||
let directive = format!(
|
||||
"notify_webhook:transformer enable=off endpoint=\"http://127.0.0.1:9/transform\" auth_token=\"secret-token\" queue_dir=\"{queue_dir}\""
|
||||
);
|
||||
let disable_response = signed_request(
|
||||
http::Method::PUT,
|
||||
&config_url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
Some(directive.into_bytes()),
|
||||
Some("text/plain"),
|
||||
)
|
||||
.await?;
|
||||
wait_for_target_visibility(&env, "transformer").await?;
|
||||
let disable_status = disable_response.status();
|
||||
let disable_body = disable_response.text().await?;
|
||||
assert_eq!(disable_status, StatusCode::OK, "failed to disable target: {disable_body}");
|
||||
|
||||
let lambda_url = format!("{}/{}/{}?lambdaArn={}", env.url, bucket, key, urlencoding::encode(lambda_arn));
|
||||
let response = signed_request(http::Method::GET, &lambda_url, &env.access_key, &env.secret_key, None, None).await?;
|
||||
@@ -1021,7 +1030,8 @@ async fn test_configure_object_lambda_target_rejects_invalid_endpoint() -> Resul
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
|
||||
.await?;
|
||||
|
||||
let bucket = "object-lambda-e2e-invalid-endpoint";
|
||||
|
||||
@@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")])
|
||||
.await?;
|
||||
|
||||
let response = send_configure_webhook_target_request(
|
||||
&env,
|
||||
@@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box
|
||||
init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
|
||||
cluster.set_env(ENV_NOTIFY_ENABLE, "true");
|
||||
cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1");
|
||||
cluster.start().await?;
|
||||
|
||||
let bucket = "listen-notification-cluster";
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
use crate::common::{RustFSTestClusterEnvironment, init_logging};
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::CompletedMultipartUpload;
|
||||
use tokio::time::{Duration, sleep};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
@@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload(
|
||||
}
|
||||
}
|
||||
|
||||
async fn complete_reports_missing_upload(
|
||||
async fn multipart_listing_reports_missing_upload(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
upload_id: &str,
|
||||
) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let result = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().build())
|
||||
.send()
|
||||
.await;
|
||||
match result {
|
||||
Ok(_) => Ok(false),
|
||||
Err(SdkError::ServiceError(err)) => {
|
||||
let code = err.err().meta().code().unwrap_or("");
|
||||
if code == "NoSuchUpload" {
|
||||
Ok(true)
|
||||
} else {
|
||||
Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into())
|
||||
}
|
||||
}
|
||||
Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()),
|
||||
}
|
||||
let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?;
|
||||
|
||||
Ok(!result
|
||||
.uploads()
|
||||
.iter()
|
||||
.any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id)))
|
||||
}
|
||||
|
||||
async fn wait_for_cleanup_on_all_nodes(
|
||||
@@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes(
|
||||
let mut all_cleaned = true;
|
||||
for (idx, client) in clients.iter().enumerate() {
|
||||
let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?;
|
||||
let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?;
|
||||
if !(list_parts_missing && complete_missing) {
|
||||
let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?;
|
||||
if !(list_parts_missing && listing_missing) {
|
||||
info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1);
|
||||
all_cleaned = false;
|
||||
break;
|
||||
@@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster()
|
||||
1,
|
||||
"multipart upload should be visible before background cleanup"
|
||||
);
|
||||
assert!(
|
||||
!multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?,
|
||||
"multipart upload listing should contain the upload before background cleanup"
|
||||
);
|
||||
|
||||
wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?;
|
||||
|
||||
|
||||
@@ -317,8 +317,6 @@ pub mod config {
|
||||
}
|
||||
|
||||
pub mod data_usage {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_usage::seed_bucket_usage_memory_for_test;
|
||||
pub use crate::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
|
||||
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
|
||||
@@ -330,6 +328,8 @@ pub mod data_usage {
|
||||
remove_bucket_usage_from_backend, replace_bucket_usage_memory_from_info, store_compression_total_in_backend,
|
||||
store_data_usage_in_backend,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::data_usage::{get_bucket_usage_memory, seed_bucket_usage_memory_for_test};
|
||||
}
|
||||
|
||||
pub mod disk {
|
||||
|
||||
@@ -3425,6 +3425,44 @@ mod tests {
|
||||
assert!(mutexes.contains_key("second"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_all_targets_publishes_disable_proxy_on_target_client() {
|
||||
// The read-proxy selector (replication_proxy::get_proxy_targets) skips
|
||||
// targets whose TargetClient carries disable_proxy — the persisted
|
||||
// per-target opt-out must survive client publication.
|
||||
let sys = BucketTargetSys::default();
|
||||
let target = |arn: &str, disable_proxy: bool| BucketTarget {
|
||||
arn: arn.to_string(),
|
||||
endpoint: "192.168.1.10:9000".to_string(),
|
||||
target_bucket: "target-bucket".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
disable_proxy,
|
||||
credentials: Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: None,
|
||||
expiration: None,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let targets = BucketTargets {
|
||||
targets: vec![target("arn:proxied", false), target("arn:opted-out", true)],
|
||||
};
|
||||
|
||||
sys.update_all_targets("bucket", Some(&targets)).await;
|
||||
|
||||
let proxied = sys
|
||||
.get_remote_target_client("bucket", "arn:proxied")
|
||||
.await
|
||||
.expect("client should be published");
|
||||
assert!(!proxied.disable_proxy);
|
||||
let opted_out = sys
|
||||
.get_remote_target_client("bucket", "arn:opted-out")
|
||||
.await
|
||||
.expect("client should be published");
|
||||
assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn target_updates_serialize_client_build_through_publication_per_bucket() {
|
||||
let sys = Arc::new(BucketTargetSys::default());
|
||||
|
||||
@@ -9136,6 +9136,7 @@ mod tests {
|
||||
assert_eq!(loaded.report.scanned, 37);
|
||||
assert_eq!(loaded.report.eligible, 11);
|
||||
assert_eq!(loaded.report.enqueued, 5);
|
||||
assert_eq!(loaded.cursor_revision, Some(1));
|
||||
assert!(loaded.lease_expires_at_unix_nanos > 0);
|
||||
let token = loaded
|
||||
.report
|
||||
@@ -9154,6 +9155,30 @@ mod tests {
|
||||
assert_eq!(admission.lease_id, loaded.lease_id);
|
||||
assert_eq!(admission.lease_expires_at_unix_nanos, loaded.lease_expires_at_unix_nanos);
|
||||
|
||||
let mut same_marker_report = report.clone();
|
||||
same_marker_report.scanned += 1;
|
||||
persist_manual_transition_page_checkpoint(
|
||||
&checkpoint_options,
|
||||
&same_marker_report,
|
||||
Some("logs/page-end".to_string()),
|
||||
Some("opaque-next-version".to_string()),
|
||||
)
|
||||
.await
|
||||
.expect("same-marker version checkpoint should persist through the durable progress sink");
|
||||
let same_marker_checkpointed = load_manual_transition_job_record(ecstore.clone(), job_id)
|
||||
.await
|
||||
.expect("same-marker version checkpoint should reload");
|
||||
assert_eq!(same_marker_checkpointed.cursor_revision, Some(2));
|
||||
let (_, version_marker) = decode_manual_transition_continuation_token(
|
||||
same_marker_checkpointed
|
||||
.report
|
||||
.continuation_token
|
||||
.as_deref()
|
||||
.expect("same-marker version checkpoint should persist a cursor"),
|
||||
)
|
||||
.expect("same-marker version cursor should decode");
|
||||
assert_eq!(version_marker.as_deref(), Some("opaque-next-version"));
|
||||
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let lifecycle_xml = format!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
@@ -9210,6 +9235,7 @@ mod tests {
|
||||
assert_eq!(checkpointed.report.scanned, 1000);
|
||||
assert_eq!(checkpointed.report.eligible, 1000);
|
||||
assert_eq!(checkpointed.report.dry_run_eligible, 1000);
|
||||
assert_eq!(checkpointed.cursor_revision, Some(3));
|
||||
let token = checkpointed
|
||||
.report
|
||||
.continuation_token
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -24,6 +24,10 @@ use crate::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport,
|
||||
};
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::durable_namespace::{
|
||||
MANUAL_TRANSITION_JOB_NAMESPACE, MANUAL_TRANSITION_SCOPE_NAMESPACE, MANUAL_TRANSITION_TASK_NAMESPACE,
|
||||
MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE,
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result as EcstoreResult};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -34,10 +38,10 @@ use crate::store::ECStore;
|
||||
pub const MANUAL_TRANSITION_JOB_SCHEMA: &str = "rustfs-manual-transition-job-v1";
|
||||
pub const MANUAL_TRANSITION_TASK_SCHEMA: &str = "rustfs-manual-transition-task-v1";
|
||||
pub const MANUAL_TRANSITION_WORKER_RESULT_SCHEMA: &str = "rustfs-manual-transition-worker-result-v1";
|
||||
pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = "ilm/manual-transition/jobs";
|
||||
pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = "ilm/manual-transition/scopes";
|
||||
pub const MANUAL_TRANSITION_TASK_PREFIX: &str = "ilm/manual-transition/tasks";
|
||||
pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = "ilm/manual-transition/results";
|
||||
pub const MANUAL_TRANSITION_JOB_RECORD_PREFIX: &str = MANUAL_TRANSITION_JOB_NAMESPACE.prefix;
|
||||
pub const MANUAL_TRANSITION_SCOPE_RECORD_PREFIX: &str = MANUAL_TRANSITION_SCOPE_NAMESPACE.prefix;
|
||||
pub const MANUAL_TRANSITION_TASK_PREFIX: &str = MANUAL_TRANSITION_TASK_NAMESPACE.prefix;
|
||||
pub const MANUAL_TRANSITION_WORKER_RESULT_PREFIX: &str = MANUAL_TRANSITION_WORKER_RESULT_NAMESPACE.prefix;
|
||||
pub const MAX_MANUAL_TRANSITION_JOB_RECORD_SIZE: usize = 64 * 1024;
|
||||
pub const MAX_MANUAL_TRANSITION_TASK_RECORD_SIZE: usize = 16 * 1024;
|
||||
pub const MAX_MANUAL_TRANSITION_WORKER_RESULT_RECORD_SIZE: usize = 8 * 1024;
|
||||
@@ -195,6 +199,8 @@ pub struct ManualTransitionJobRecord {
|
||||
pub updated_at_unix_nanos: i128,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub completed_at_unix_nanos: Option<i128>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub cursor_revision: Option<u64>,
|
||||
pub report: ManualTransitionRunReport,
|
||||
pub queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -224,6 +230,7 @@ impl ManualTransitionJobRecord {
|
||||
created_at_unix_nanos: now,
|
||||
updated_at_unix_nanos: now,
|
||||
completed_at_unix_nanos: None,
|
||||
cursor_revision: Some(0),
|
||||
report: ManualTransitionRunReport {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: options.prefix.clone(),
|
||||
@@ -238,7 +245,7 @@ impl ManualTransitionJobRecord {
|
||||
|
||||
pub fn complete(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
|
||||
self.scan_completed = true;
|
||||
self.report.merge_scan_report_preserving_worker(&report);
|
||||
self.merge_scan_report(&report);
|
||||
self.queue_snapshot = queue_snapshot;
|
||||
self.error = None;
|
||||
self.mark_terminal_if_worker_drained();
|
||||
@@ -316,7 +323,7 @@ impl ManualTransitionJobRecord {
|
||||
}
|
||||
}
|
||||
self.queue_snapshot = queue_snapshot;
|
||||
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
|
||||
self.advance_updated_at();
|
||||
self.mark_terminal_if_worker_drained();
|
||||
}
|
||||
|
||||
@@ -359,14 +366,14 @@ impl ManualTransitionJobRecord {
|
||||
self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
|
||||
self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
|
||||
self.queue_snapshot = queue_snapshot;
|
||||
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
|
||||
self.advance_updated_at();
|
||||
self.mark_terminal_if_worker_drained();
|
||||
true
|
||||
}
|
||||
|
||||
pub fn mark_cancel_requested(&mut self) {
|
||||
self.cancel_requested = true;
|
||||
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
|
||||
self.advance_updated_at();
|
||||
}
|
||||
|
||||
pub fn claim_recovery_lease(&mut self, owner_id: impl Into<String>, queue_snapshot: ManualTransitionQueueSnapshot) {
|
||||
@@ -380,7 +387,7 @@ impl ManualTransitionJobRecord {
|
||||
pub fn abandon_recovery_lease(&mut self, lease_id: Uuid) {
|
||||
if self.state == ManualTransitionJobState::Running && self.lease_id == lease_id {
|
||||
self.lease_expires_at_unix_nanos = 0;
|
||||
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
|
||||
self.advance_updated_at();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -398,7 +405,7 @@ impl ManualTransitionJobRecord {
|
||||
|
||||
pub fn renew_lease(&mut self, queue_snapshot: ManualTransitionQueueSnapshot) {
|
||||
let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
|
||||
self.updated_at_unix_nanos = now;
|
||||
self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
|
||||
self.lease_expires_at_unix_nanos = manual_transition_job_lease_expires_at(now);
|
||||
self.queue_snapshot = queue_snapshot;
|
||||
}
|
||||
@@ -438,11 +445,18 @@ impl ManualTransitionJobRecord {
|
||||
|
||||
pub fn update_running_progress(&mut self, report: ManualTransitionRunReport, queue_snapshot: ManualTransitionQueueSnapshot) {
|
||||
if self.state == ManualTransitionJobState::Running {
|
||||
self.report.merge_scan_report_preserving_worker(&report);
|
||||
self.merge_scan_report(&report);
|
||||
self.renew_lease(queue_snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_scan_report(&mut self, report: &ManualTransitionRunReport) {
|
||||
if self.report.continuation_token != report.continuation_token {
|
||||
self.cursor_revision = Some(self.cursor_revision.unwrap_or(0).saturating_add(1));
|
||||
}
|
||||
self.report.merge_scan_report_preserving_worker(report);
|
||||
}
|
||||
|
||||
pub fn mark_unknown_if_unowned(&mut self) {
|
||||
if self.state == ManualTransitionJobState::Running {
|
||||
self.state = ManualTransitionJobState::Unknown;
|
||||
@@ -463,9 +477,13 @@ impl ManualTransitionJobRecord {
|
||||
}
|
||||
|
||||
fn mark_updated_terminal(&mut self) {
|
||||
self.advance_updated_at();
|
||||
self.completed_at_unix_nanos = Some(self.updated_at_unix_nanos);
|
||||
}
|
||||
|
||||
fn advance_updated_at(&mut self) {
|
||||
let now = OffsetDateTime::now_utc().unix_timestamp_nanos();
|
||||
self.updated_at_unix_nanos = now;
|
||||
self.completed_at_unix_nanos = Some(now);
|
||||
self.updated_at_unix_nanos = self.updated_at_unix_nanos.saturating_add(1).max(now);
|
||||
}
|
||||
|
||||
fn mark_terminal_if_worker_drained(&mut self) {
|
||||
@@ -1109,7 +1127,8 @@ pub fn manual_transition_scope_record_object_name(scope_key: &str) -> Result<Str
|
||||
pub async fn save_manual_transition_job_record(api: Arc<ECStore>, job: &ManualTransitionJobRecord) -> EcstoreResult<()> {
|
||||
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
|
||||
let data = job.encode().map_err(manual_transition_job_store_error)?;
|
||||
config_boundary::save_config(api, &object, data).await
|
||||
config_boundary::save_config(api.clone(), &object, data.clone()).await?;
|
||||
api.record_durable_ilm_decommission_progress(&object, &data).await
|
||||
}
|
||||
|
||||
pub async fn load_manual_transition_job_record(api: Arc<ECStore>, job_id: Uuid) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
@@ -1142,9 +1161,9 @@ pub async fn save_manual_transition_job_record_if_current(
|
||||
let object = manual_transition_job_record_object_name(job.job_id).map_err(manual_transition_job_store_error)?;
|
||||
let data = job.encode().map_err(manual_transition_job_store_error)?;
|
||||
config_boundary::save_config_with_opts_quiet(
|
||||
api,
|
||||
api.clone(),
|
||||
&object,
|
||||
data,
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
@@ -1154,7 +1173,8 @@ pub async fn save_manual_transition_job_record_if_current(
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
api.record_durable_ilm_decommission_progress(&object, &data).await
|
||||
}
|
||||
|
||||
/// Applies a job-record mutation with optimistic concurrency control.
|
||||
@@ -1592,9 +1612,9 @@ pub async fn save_manual_transition_scope_admission_if_absent(
|
||||
let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
|
||||
let data = serde_json::to_vec(admission).map_err(Error::other)?;
|
||||
config_boundary::save_config_with_opts(
|
||||
api,
|
||||
api.clone(),
|
||||
&object,
|
||||
data,
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
@@ -1604,7 +1624,8 @@ pub async fn save_manual_transition_scope_admission_if_absent(
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
api.record_durable_ilm_decommission_progress(&object, &data).await
|
||||
}
|
||||
|
||||
pub async fn load_manual_transition_scope_admission(
|
||||
@@ -1642,9 +1663,9 @@ pub async fn save_manual_transition_scope_admission_if_current(
|
||||
let object = manual_transition_scope_record_object_name(&admission.scope_key).map_err(manual_transition_job_store_error)?;
|
||||
let data = serde_json::to_vec(admission).map_err(Error::other)?;
|
||||
match config_boundary::save_config_with_opts(
|
||||
api,
|
||||
api.clone(),
|
||||
&object,
|
||||
data,
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
@@ -1660,7 +1681,8 @@ pub async fn save_manual_transition_scope_admission_if_current(
|
||||
Err(Error::PreconditionFailed)
|
||||
}
|
||||
result => result,
|
||||
}
|
||||
}?;
|
||||
api.record_durable_ilm_decommission_progress(&object, &data).await
|
||||
}
|
||||
|
||||
pub async fn claim_manual_transition_scope_admission(
|
||||
@@ -1953,13 +1975,15 @@ pub async fn delete_manual_transition_scope_admission_if_current(
|
||||
job_id: Uuid,
|
||||
lease_id: Uuid,
|
||||
) -> EcstoreResult<bool> {
|
||||
let etag = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
|
||||
Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => etag,
|
||||
let (admission, etag) = match load_manual_transition_scope_admission_with_etag(api.clone(), scope_key).await {
|
||||
Ok((admission, etag)) if admission.job_id == job_id && admission.lease_id == lease_id => (admission, etag),
|
||||
Ok(_) => return Ok(false),
|
||||
Err(Error::ConfigNotFound) => return Ok(true),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let object = manual_transition_scope_record_object_name(scope_key).map_err(manual_transition_job_store_error)?;
|
||||
let data = serde_json::to_vec(&admission).map_err(Error::other)?;
|
||||
api.record_durable_ilm_decommission_terminal(&object, &data).await?;
|
||||
match config_boundary::delete_config_if_match(api, &object, &etag).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound) => Ok(true),
|
||||
Err(Error::PreconditionFailed) => Ok(false),
|
||||
@@ -2577,6 +2601,25 @@ mod tests {
|
||||
assert!(decoded.report.tier_failure_by_reason.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_record_decodes_legacy_cursor_without_revision() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
let record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||
let encoded = record.encode().expect("job record should encode");
|
||||
let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
|
||||
value["job"]
|
||||
.as_object_mut()
|
||||
.expect("job should be object")
|
||||
.remove("cursor_revision");
|
||||
let record_bytes = serde_json::to_vec(&value["job"]).expect("legacy job should encode");
|
||||
value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
|
||||
let legacy = serde_json::to_vec(&value).expect("legacy envelope should encode");
|
||||
|
||||
let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job should decode");
|
||||
|
||||
assert_eq!(decoded.cursor_revision, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_record_rejects_unknown_report_fields() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
|
||||
@@ -16,6 +16,7 @@ pub mod bucket_lifecycle_audit;
|
||||
pub mod bucket_lifecycle_ops;
|
||||
mod config_boundary;
|
||||
pub mod core;
|
||||
mod durable_namespace;
|
||||
pub mod evaluator;
|
||||
pub mod manual_transition_job;
|
||||
mod metadata_boundary;
|
||||
@@ -31,3 +32,8 @@ pub mod tier_free_version_recovery;
|
||||
pub mod tier_last_day_stats;
|
||||
pub mod tier_sweeper;
|
||||
pub mod transition_transaction;
|
||||
|
||||
pub(crate) use durable_namespace::{
|
||||
DurableIlmRecordCheckpoint, ILM_META_PREFIX, ValidatedDurableIlmRecord, classify_durable_ilm_record,
|
||||
validate_durable_ilm_record,
|
||||
};
|
||||
|
||||
@@ -20,6 +20,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::durable_namespace::TIER_DELETE_JOURNAL_NAMESPACE;
|
||||
use crate::bucket::lifecycle::runtime_boundary;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
Jentry, TierDeleteJournalState, TierDeleteSourceIdentity,
|
||||
@@ -49,7 +50,7 @@ const TIER_DELETE_JOURNAL_VERSION: u8 = 2;
|
||||
const TIER_DELETE_JOURNAL_EXACT_VERSION: u8 = 3;
|
||||
const TIER_DELETE_JOURNAL_STATE_VERSION: u8 = 4;
|
||||
const TIER_DELETE_JOURNAL_TRANSACTION_VERSION: u8 = 5;
|
||||
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = "ilm/tier-delete-journal/";
|
||||
pub(crate) const TIER_DELETE_JOURNAL_PREFIX: &str = TIER_DELETE_JOURNAL_NAMESPACE.prefix;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -432,6 +433,21 @@ async fn process_committed_tier_delete_journal_entry(api: Arc<ECStore>, je: &Jen
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
let path = tier_delete_journal_object_name(je);
|
||||
let data = encode_tier_delete_journal_entry(je).map_err(std::io::Error::other)?;
|
||||
let target_pool_indices = api
|
||||
.record_durable_ilm_decommission_terminal_target_pools(&path, &data)
|
||||
.await
|
||||
.map_err(std::io::Error::other)?;
|
||||
if let Some(target_pool_indices) = target_pool_indices {
|
||||
for target_pool_idx in target_pool_indices {
|
||||
match config_boundary::delete_config(api.pools[target_pool_idx].clone(), &path).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound) => {}
|
||||
Err(err) => return Err(std::io::Error::other(err)),
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
remove_tier_delete_journal_entry(api, je).await
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ use tracing::{debug, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::durable_namespace::TRANSITION_TRANSACTION_NAMESPACE;
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::lifecycle::tier_sweeper::{
|
||||
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
|
||||
@@ -42,7 +43,7 @@ const TRANSITION_TRANSACTION_RECOVERY_INTERVAL: Duration = Duration::from_secs(6
|
||||
const TRANSITION_TRANSACTION_RECOVERY_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
pub const TRANSITION_TRANSACTION_SCHEMA: &str = "rustfs-transition-transaction-v1";
|
||||
pub const TRANSITION_TRANSACTION_PREFIX: &str = "ilm/transition-transactions";
|
||||
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = "ilm/transition-transactions/records";
|
||||
pub const TRANSITION_TRANSACTION_RECORD_PREFIX: &str = TRANSITION_TRANSACTION_NAMESPACE.prefix;
|
||||
pub const MAX_TRANSITION_TRANSACTION_SIZE: usize = 64 * 1024;
|
||||
|
||||
pub type Result<T> = std::result::Result<T, TransitionTransactionError>;
|
||||
@@ -584,7 +585,8 @@ pub(crate) async fn save_transition_transaction_record(
|
||||
let object =
|
||||
transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
|
||||
let data = transaction.encode().map_err(transition_transaction_store_error)?;
|
||||
config_boundary::save_config(api, &object, data).await
|
||||
config_boundary::save_config(api.clone(), &object, data.clone()).await?;
|
||||
api.record_durable_ilm_decommission_progress(&object, &data).await
|
||||
}
|
||||
|
||||
pub(crate) async fn load_transition_transaction_record(
|
||||
@@ -596,8 +598,14 @@ pub(crate) async fn load_transition_transaction_record(
|
||||
TransitionTransaction::decode(transaction_id, &data).map_err(transition_transaction_store_error)
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_transition_transaction_record(api: Arc<ECStore>, transaction_id: Uuid) -> EcstoreResult<()> {
|
||||
let object = transition_transaction_record_object_name(transaction_id).map_err(transition_transaction_store_error)?;
|
||||
pub(crate) async fn delete_transition_transaction_record(
|
||||
api: Arc<ECStore>,
|
||||
transaction: &TransitionTransaction,
|
||||
) -> EcstoreResult<()> {
|
||||
let object =
|
||||
transition_transaction_record_object_name(transaction.transaction_id).map_err(transition_transaction_store_error)?;
|
||||
let data = transaction.encode().map_err(transition_transaction_store_error)?;
|
||||
api.record_durable_ilm_decommission_terminal(&object, &data).await?;
|
||||
match config_boundary::delete_config(api, &object).await {
|
||||
Ok(()) | Err(Error::ConfigNotFound) => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
@@ -813,7 +821,7 @@ pub async fn finalize_missing_transition_transaction_for_operator(
|
||||
if probe != TransitionOperatorProbe::Missing {
|
||||
return Err(TransitionOperatorError::CandidateNotMissing(probe));
|
||||
}
|
||||
delete_transition_transaction_record(api, transaction_id)
|
||||
delete_transition_transaction_record(api, &transaction)
|
||||
.await
|
||||
.map_err(TransitionOperatorError::Store)
|
||||
}
|
||||
@@ -849,22 +857,22 @@ pub async fn process_transition_transaction_record(
|
||||
match transaction.state {
|
||||
TransitionTransactionState::Uploaded => {
|
||||
delete_transition_remote_candidate(api.clone(), transaction).await?;
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
TransitionTransactionState::CleanupPending => match local_commit_matches_transaction(api.clone(), transaction).await {
|
||||
Ok(true) => {
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
Ok(false) => {
|
||||
delete_transition_remote_candidate(api.clone(), transaction).await?;
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
Err(err) if transition_source_is_missing(&err) => {
|
||||
delete_transition_remote_candidate(api.clone(), transaction).await?;
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
@@ -872,7 +880,7 @@ pub async fn process_transition_transaction_record(
|
||||
TransitionTransactionState::LocalCommitStarted => {
|
||||
match local_commit_matches_transaction(api.clone(), transaction).await {
|
||||
Ok(true) => {
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
Ok(false) => Ok(TransitionTransactionRecoveryOutcome::Retained),
|
||||
@@ -881,7 +889,7 @@ pub async fn process_transition_transaction_record(
|
||||
}
|
||||
}
|
||||
TransitionTransactionState::AbortedNoRemote | TransitionTransactionState::Committed => {
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
TransitionTransactionState::UploadOutcomeUnknown => recover_unknown_upload_outcome(api, transaction).await,
|
||||
@@ -907,7 +915,7 @@ async fn recover_unknown_upload_outcome(
|
||||
.map_err(Error::other)?
|
||||
{
|
||||
TransitionCandidateProbe::Missing => {
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RecordDeleted)
|
||||
}
|
||||
TransitionCandidateProbe::UnversionedPresent => {
|
||||
@@ -925,7 +933,7 @@ async fn recover_unknown_upload_outcome(
|
||||
)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
delete_transition_transaction_record(api, transaction.transaction_id).await?;
|
||||
delete_transition_transaction_record(api, transaction).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
TransitionCandidateProbe::VersionedPresent(version_id) => {
|
||||
@@ -958,7 +966,7 @@ async fn cleanup_recovered_unknown_upload_candidate(
|
||||
.map_err(transition_transaction_store_error)?;
|
||||
save_transition_transaction_record(api.clone(), &cleanup).await?;
|
||||
delete_transition_remote_candidate(api.clone(), &cleanup).await?;
|
||||
delete_transition_transaction_record(api, cleanup.transaction_id).await?;
|
||||
delete_transition_transaction_record(api, &cleanup).await?;
|
||||
Ok(TransitionTransactionRecoveryOutcome::RemoteCandidateDeleted)
|
||||
}
|
||||
|
||||
|
||||
@@ -2855,7 +2855,7 @@ fn replicate_object_info_from_object_info(
|
||||
.map(|v| OffsetDateTime::parse(&v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
|
||||
let mut rstate = oi.replication_state();
|
||||
rstate.replicate_decision_str = dsc.to_string();
|
||||
let asz = oi.get_actual_size().unwrap_or_default();
|
||||
let asz = oi.get_actual_size_or_physical();
|
||||
let ssec = replication_object_is_ssec_encrypted(&oi.user_defined);
|
||||
let checksum = if ssec { oi.checksum.clone() } else { None };
|
||||
|
||||
|
||||
@@ -1412,7 +1412,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
};
|
||||
let mut replication_state = oi.replication_state();
|
||||
replication_state.replicate_decision_str = dsc.to_string();
|
||||
let actual_size = oi.get_actual_size().unwrap_or_default();
|
||||
let actual_size = oi.get_actual_size_or_physical();
|
||||
|
||||
Ok(ReplicateObjectInfo {
|
||||
name: oi.name.clone(),
|
||||
|
||||
@@ -389,7 +389,7 @@ fn replication_source_object(object_info: &ObjectInfo) -> ReplicationSourceObjec
|
||||
.map(|mod_time| OffsetDateTime::from_unix_timestamp(mod_time.unix_timestamp()).unwrap_or(mod_time)),
|
||||
version_id: object_info.version_id.map(|version_id| version_id.to_string()),
|
||||
etag: object_info.etag.as_deref(),
|
||||
actual_size: object_info.get_actual_size().unwrap_or_default(),
|
||||
actual_size: object_info.get_actual_size_or_physical(),
|
||||
delete_marker: object_info.delete_marker,
|
||||
content_type: object_info.content_type.as_deref(),
|
||||
content_encoding: object_info.content_encoding.as_deref(),
|
||||
@@ -542,6 +542,20 @@ mod tests {
|
||||
assert!(replication_target_head_is_newer_null_version(&source, &target));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_source_uses_physical_size_for_unknown_compressed_object() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "zstd".to_string());
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
actual_size: -1,
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(replication_source_object(&source).actual_size, 128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_target_head_content_matches_compare_etag_only() {
|
||||
let source = ObjectInfo {
|
||||
|
||||
@@ -42,18 +42,19 @@ use futures::lock::Mutex;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE,
|
||||
INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
|
||||
INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
|
||||
INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
|
||||
INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP,
|
||||
};
|
||||
use rustfs_protos::ChannelClass;
|
||||
use rustfs_protos::evict_failed_connection;
|
||||
use rustfs_protos::proto_gen::node_service::RenamePartRequest;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, DeletePathsRequest, DeleteRequest,
|
||||
DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest,
|
||||
ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest,
|
||||
RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
DeleteVersionRequest, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
@@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50);
|
||||
const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ";
|
||||
const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC";
|
||||
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
|
||||
const BATCH_METADATA_RPC_OFF: &str = "off";
|
||||
const BATCH_METADATA_RPC_AUTO: &str = "auto";
|
||||
const BATCH_METADATA_RPC_ON: &str = "on";
|
||||
@@ -112,6 +114,28 @@ const EVENT_REMOTE_DISK_RPC: &str = "remote_disk_rpc";
|
||||
const SNAPSHOT_LEASE_PROTOCOL_VERSION: u32 = 1;
|
||||
pub const REMOTE_SNAPSHOT_LEASE_TTL: Duration = Duration::from_secs(60);
|
||||
|
||||
fn decode_delete_versions_errors(response: DeleteVersionsResponse, expected_len: usize) -> Vec<Option<Error>> {
|
||||
if !response.item_errors.is_empty() {
|
||||
if response.item_errors.len() != expected_len {
|
||||
return vec![Some(Error::other("malformed delete_versions item errors")); expected_len];
|
||||
}
|
||||
return response
|
||||
.item_errors
|
||||
.into_iter()
|
||||
.map(|error| (error.code != 0).then(|| error.into()))
|
||||
.collect();
|
||||
}
|
||||
|
||||
if response.errors.len() != expected_len {
|
||||
return vec![Some(Error::other("malformed delete_versions errors")); expected_len];
|
||||
}
|
||||
response
|
||||
.errors
|
||||
.into_iter()
|
||||
.map(|error| (!error.is_empty()).then(|| Error::other(error)))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn snapshot_lease_token_from_response(response: SnapshotLeaseResponse) -> Result<SnapshotLeaseToken> {
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
@@ -180,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode {
|
||||
}
|
||||
|
||||
fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode {
|
||||
rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)
|
||||
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
|
||||
.or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ))
|
||||
.or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC))
|
||||
.as_deref()
|
||||
.map(parse_batch_metadata_rpc_mode)
|
||||
@@ -1804,6 +1829,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
|
||||
}
|
||||
}
|
||||
|
||||
fn record_batch_read_version_stage(stage: &'static str, started_at: Option<Instant>) {
|
||||
if let Some(started_at) = started_at {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed());
|
||||
}
|
||||
}
|
||||
|
||||
/// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads
|
||||
/// and falling back to the JSON compatibility strings. Used to size the RPC for the payload
|
||||
/// histogram / large-payload alerting (grpc-optimization P0 instrumentation).
|
||||
@@ -1914,6 +1945,27 @@ fn decode_batch_read_version_response_items(
|
||||
Ok(batch_read_version_resps)
|
||||
}
|
||||
|
||||
fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize {
|
||||
req.items
|
||||
.iter()
|
||||
.fold(req_json.len().saturating_add(req_bin.len()), |total, item| {
|
||||
total
|
||||
.saturating_add(item.org_volume.len())
|
||||
.saturating_add(item.volume.len())
|
||||
.saturating_add(item.path.len())
|
||||
.saturating_add(item.version_id.len())
|
||||
})
|
||||
}
|
||||
|
||||
fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize {
|
||||
response
|
||||
.batch_read_version_resps
|
||||
.iter()
|
||||
.map(String::len)
|
||||
.sum::<usize>()
|
||||
.saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::<usize>())
|
||||
}
|
||||
|
||||
fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
file_info.validate_for_metadata_read().map_err(Into::into)
|
||||
}
|
||||
@@ -2406,8 +2458,6 @@ impl DiskAPI for RemoteDisk {
|
||||
return errors;
|
||||
}
|
||||
|
||||
// TODO(backlog): replace string errors with typed `StorageError` variants
|
||||
|
||||
let result = self
|
||||
.execute_with_timeout(
|
||||
|| async {
|
||||
@@ -2439,17 +2489,7 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
response
|
||||
.errors
|
||||
.iter()
|
||||
.map(|error| {
|
||||
if error.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Error::other(error.to_string()))
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
decode_delete_versions_errors(response, versions.len())
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -2827,14 +2867,19 @@ impl DiskAPI for RemoteDisk {
|
||||
state = "started",
|
||||
"Remote disk RPC started"
|
||||
);
|
||||
let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
|
||||
let batch_read_version_req = compat_json(&req)?;
|
||||
let batch_read_version_req_bin = encode_msgpack(&req)?;
|
||||
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
|
||||
let request_payload_bytes = batch_read_version_attribution_enabled
|
||||
.then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
|
||||
let batch_result = self
|
||||
.execute_with_timeout_for_op(
|
||||
"batch_read_version",
|
||||
move || async move {
|
||||
let disk = self.disk_ref().await;
|
||||
let disk_len = disk.len();
|
||||
let mut client = self
|
||||
.get_bulk_client()
|
||||
.await
|
||||
@@ -2845,9 +2890,20 @@ impl DiskAPI for RemoteDisk {
|
||||
batch_read_version_req_bin: batch_read_version_req_bin.into(),
|
||||
});
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
|
||||
if let Some(request_payload_bytes) = request_payload_bytes {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
|
||||
request_payload_bytes.saturating_add(disk_len),
|
||||
);
|
||||
}
|
||||
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
|
||||
let response = match client.batch_read_version(request).await {
|
||||
Ok(response) => response.into_inner(),
|
||||
Ok(response) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
response.into_inner()
|
||||
}
|
||||
Err(status) if status.code() == Code::Unimplemented => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
if mode.should_fallback_on_unimplemented() {
|
||||
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED);
|
||||
warn!(
|
||||
@@ -2864,6 +2920,7 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
|
||||
record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -2876,14 +2933,33 @@ impl DiskAPI for RemoteDisk {
|
||||
);
|
||||
return Err(Error::from(status));
|
||||
}
|
||||
Err(status) => return Err(Error::from(status)),
|
||||
Err(status) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
|
||||
return Err(Error::from(status));
|
||||
}
|
||||
};
|
||||
|
||||
if !response.success {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
|
||||
decode_batch_read_version_response_items(response, &self.endpoint).map(Some)
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes(
|
||||
batch_read_version_response_payload_len(&response),
|
||||
);
|
||||
let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled);
|
||||
match decode_batch_read_version_response_items(response, &self.endpoint) {
|
||||
Ok(batch_read_version_resps) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
|
||||
Ok(Some(batch_read_version_resps))
|
||||
}
|
||||
Err(err) => {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started);
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error();
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
@@ -3760,6 +3836,63 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
#[test]
|
||||
fn delete_versions_response_preserves_typed_item_errors() {
|
||||
let errors = decode_delete_versions_errors(
|
||||
DeleteVersionsResponse {
|
||||
success: true,
|
||||
errors: vec!["file not found".to_string(), String::new()],
|
||||
error: None,
|
||||
item_errors: vec![
|
||||
rustfs_protos::proto_gen::node_service::Error {
|
||||
code: DiskError::FileNotFound.to_u32(),
|
||||
error_info: "file not found".to_string(),
|
||||
},
|
||||
rustfs_protos::proto_gen::node_service::Error::default(),
|
||||
],
|
||||
},
|
||||
2,
|
||||
);
|
||||
|
||||
assert!(matches!(errors.as_slice(), [Some(DiskError::FileNotFound), None]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_versions_response_accepts_legacy_string_errors() {
|
||||
let errors = decode_delete_versions_errors(
|
||||
DeleteVersionsResponse {
|
||||
success: true,
|
||||
errors: vec!["legacy error".to_string(), String::new()],
|
||||
error: None,
|
||||
item_errors: Vec::new(),
|
||||
},
|
||||
2,
|
||||
);
|
||||
|
||||
assert_eq!(errors.len(), 2);
|
||||
assert_eq!(errors[0].as_ref().map(ToString::to_string).as_deref(), Some("io error legacy error"));
|
||||
assert!(errors[1].is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_versions_response_rejects_misaligned_item_errors() {
|
||||
let errors = decode_delete_versions_errors(
|
||||
DeleteVersionsResponse {
|
||||
success: true,
|
||||
errors: vec!["file not found".to_string()],
|
||||
error: None,
|
||||
item_errors: vec![rustfs_protos::proto_gen::node_service::Error {
|
||||
code: DiskError::FileNotFound.to_u32(),
|
||||
error_info: "file not found".to_string(),
|
||||
}],
|
||||
},
|
||||
2,
|
||||
);
|
||||
|
||||
assert_eq!(errors.len(), 2);
|
||||
assert!(errors.iter().all(Option::is_some));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_mutation_digest_marks_rolling_compatibility() {
|
||||
let mut request = Request::new(());
|
||||
@@ -4554,6 +4687,7 @@ mod tests {
|
||||
} else {
|
||||
"file version not found".to_string()
|
||||
},
|
||||
error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() },
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4673,6 +4807,7 @@ mod tests {
|
||||
fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
|
||||
(ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")),
|
||||
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
|
||||
],
|
||||
@@ -4682,10 +4817,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")),
|
||||
(ENV_RUSTFS_METADATA_BATCH_READ, Some("off")),
|
||||
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")),
|
||||
],
|
||||
|| {
|
||||
assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>),
|
||||
(ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>),
|
||||
(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")),
|
||||
],
|
||||
|
||||
@@ -14,9 +14,10 @@
|
||||
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE,
|
||||
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION,
|
||||
INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics,
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
|
||||
INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM,
|
||||
INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP,
|
||||
global_internode_metrics,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() {
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_request() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_outgoing_request_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_stage_duration_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
stage,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics()
|
||||
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
bytes,
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
global_internode_metrics().record_recv_bytes_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
|
||||
INTERNODE_TRANSPORT_BACKEND_GRPC,
|
||||
bytes,
|
||||
);
|
||||
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
|
||||
}
|
||||
|
||||
pub(crate) fn record_remote_disk_grpc_read_version_error() {
|
||||
if !rustfs_io_metrics::get_stage_metrics_enabled() {
|
||||
return;
|
||||
|
||||
@@ -406,6 +406,25 @@ where
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty<S>(api: Arc<S>, file: &str, max_bytes: usize) -> Result<Vec<u8>>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let (data, _obj) = read_config_limited_preserve_empty_with_metadata(api, file, max_bytes).await?;
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config_limited_preserve_empty_with_metadata<S>(
|
||||
api: Arc<S>,
|
||||
file: &str,
|
||||
max_bytes: usize,
|
||||
) -> Result<(Vec<u8>, ObjectInfo)>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
read_config_with_metadata_inner(api, file, &ObjectOptions::default(), true, Some(max_bytes)).await
|
||||
}
|
||||
|
||||
/// Read an existing config object without treating an empty payload as absent.
|
||||
/// Callers that validate their own payload format need to distinguish corruption
|
||||
/// from `ConfigNotFound`.
|
||||
|
||||
@@ -248,10 +248,13 @@ impl Config {
|
||||
let shard_size = shard_size as usize;
|
||||
// Keep the historical two-data-shard object budget while preventing
|
||||
// wider EC layouts from multiplying the maximum inline object size.
|
||||
// Use div_ceil to match the shard_file_size calculation (which also uses
|
||||
// div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline
|
||||
// for objects right at the threshold.
|
||||
let inline_block = if self.initialized && self.inline_block_explicit {
|
||||
self.inline_block
|
||||
} else {
|
||||
(DEFAULT_INLINE_OBJECT_BUDGET / data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
};
|
||||
|
||||
if versioned {
|
||||
|
||||
+3918
-590
File diff suppressed because it is too large
Load Diff
@@ -21,7 +21,7 @@ use crate::storage_api_contracts::{
|
||||
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
|
||||
multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo},
|
||||
object::{DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
|
||||
object::{DeleteAccounting, DeletedObject, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
|
||||
range::HTTPRangeSpec,
|
||||
};
|
||||
use crate::{
|
||||
@@ -414,6 +414,66 @@ fn apply_delete_objects_results(
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_delete_accounting_results(
|
||||
accounting: &mut [Option<DeleteAccounting>],
|
||||
set_objects: &[DelObj],
|
||||
set_accounting: &[Option<DeleteAccounting>],
|
||||
) {
|
||||
for (obj, value) in set_objects.iter().zip(set_accounting.iter()) {
|
||||
accounting[obj.orig_idx] = value.clone();
|
||||
}
|
||||
}
|
||||
|
||||
impl Sets {
|
||||
pub(crate) async fn delete_objects_with_accounting(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
let mut del_errs = vec![None; objects.len()];
|
||||
let mut accounting = vec![None; objects.len()];
|
||||
let mut set_obj_map = HashMap::new();
|
||||
|
||||
for (i, obj) in objects.iter().enumerate() {
|
||||
let idx = self.get_hashed_set_index(obj.object_name.as_str());
|
||||
set_obj_map.entry(idx).or_insert_with(Vec::new).push(DelObj {
|
||||
orig_idx: i,
|
||||
obj: obj.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
|
||||
let mut futures = FuturesUnordered::new();
|
||||
let bucket = bucket.to_owned();
|
||||
|
||||
for (set_index, set_objects) in set_obj_map {
|
||||
let disks = self.get_disks(set_index);
|
||||
let objects = set_objects.iter().map(|entry| entry.obj.clone()).collect::<Vec<_>>();
|
||||
let bucket = bucket.clone();
|
||||
let opts = opts.clone();
|
||||
let semaphore = semaphore.clone();
|
||||
futures.push(async move {
|
||||
let _permit = semaphore
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("delete_objects semaphore should remain open");
|
||||
let (deleted, errors, accounting) = disks.delete_objects_with_accounting(&bucket, objects, opts).await;
|
||||
(set_objects, deleted, errors, accounting)
|
||||
});
|
||||
}
|
||||
|
||||
while let Some((set_objects, deleted, errors, set_accounting)) = futures.next().await {
|
||||
apply_delete_objects_results(&mut del_objects, &mut del_errs, &set_objects, &deleted, errors);
|
||||
apply_delete_accounting_results(&mut accounting, &set_objects, &set_accounting);
|
||||
}
|
||||
|
||||
(del_objects, del_errs, accounting)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for Sets {
|
||||
type Error = Error;
|
||||
@@ -655,65 +715,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
|
||||
// Default return value
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
|
||||
let mut del_errs = Vec::with_capacity(objects.len());
|
||||
for _ in 0..objects.len() {
|
||||
del_errs.push(None)
|
||||
}
|
||||
|
||||
let mut set_obj_map = HashMap::new();
|
||||
|
||||
// hash key
|
||||
for (i, obj) in objects.iter().enumerate() {
|
||||
let idx = self.get_hashed_set_index(obj.object_name.as_str());
|
||||
|
||||
if !set_obj_map.contains_key(&idx) {
|
||||
set_obj_map.insert(
|
||||
idx,
|
||||
vec![DelObj {
|
||||
// set_idx: idx,
|
||||
orig_idx: i,
|
||||
obj: obj.clone(),
|
||||
}],
|
||||
);
|
||||
} else if let Some(val) = set_obj_map.get_mut(&idx) {
|
||||
val.push(DelObj {
|
||||
// set_idx: idx,
|
||||
orig_idx: i,
|
||||
obj: obj.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let max_concurrent = set_obj_map.len().min(num_cpus::get()).max(1);
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent));
|
||||
let mut futures = FuturesUnordered::new();
|
||||
let bucket = bucket.to_string();
|
||||
|
||||
for (k, v) in set_obj_map {
|
||||
let disks = self.get_disks(k);
|
||||
let objs: Vec<ObjectToDelete> = v.iter().map(|v| v.obj.clone()).collect();
|
||||
let bucket = bucket.clone();
|
||||
let opts = opts.clone();
|
||||
let semaphore = semaphore.clone();
|
||||
|
||||
futures.push(async move {
|
||||
let _permit = semaphore
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("delete_objects semaphore should remain open");
|
||||
let (dobjects, errs) = disks.delete_objects(&bucket, objs, opts).await;
|
||||
(v, dobjects, errs)
|
||||
});
|
||||
}
|
||||
|
||||
while let Some((v, dobjects, errs)) = futures.next().await {
|
||||
apply_delete_objects_results(&mut del_objects, &mut del_errs, &v, &dobjects, errs);
|
||||
}
|
||||
|
||||
(del_objects, del_errs)
|
||||
let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
|
||||
(deleted, errors)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
|
||||
@@ -26,7 +26,7 @@ use crate::storage_api_contracts::{
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectOperations as _},
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence};
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex};
|
||||
@@ -856,7 +856,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo)
|
||||
fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool {
|
||||
is_unversioned_data_movement_object(source)
|
||||
&& is_unversioned_data_movement_object(target)
|
||||
&& !target.delete_marker
|
||||
&& source
|
||||
.mod_time
|
||||
.zip(target.mod_time)
|
||||
@@ -1028,6 +1027,7 @@ pub(crate) enum SourceCleanupError {
|
||||
pub(crate) struct SourceCleanupBucketFence<'a> {
|
||||
pub(crate) expected_incarnation_id: Option<uuid::Uuid>,
|
||||
pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>,
|
||||
pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>,
|
||||
}
|
||||
|
||||
fn ensure_source_cleanup_versions_match(
|
||||
@@ -1065,7 +1065,9 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged(
|
||||
struct SourceCleanupDeleteBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
fence_pending: tokio::sync::Notify,
|
||||
arrived: tokio::sync::Notify,
|
||||
is_paused: AtomicBool,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
@@ -1079,7 +1081,7 @@ pub(crate) struct SourceCleanupDeleteBarrier {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option<Arc<SourceCleanupDeleteBarrierState>>>> =
|
||||
static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<SourceCleanupDeleteBarrierState>>>> =
|
||||
std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1092,15 +1094,22 @@ impl SourceCleanupDeleteBarrier {
|
||||
let state = Arc::new(SourceCleanupDeleteBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
fence_pending: tokio::sync::Notify::new(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
is_paused: AtomicBool::new(false),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison");
|
||||
assert!(slot.is_none(), "source cleanup delete barrier must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
assert!(
|
||||
!barriers
|
||||
.iter()
|
||||
.any(|barrier| barrier.bucket == bucket && barrier.object == object),
|
||||
"source cleanup delete barrier must be unique per object"
|
||||
);
|
||||
barriers.push(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
@@ -1110,35 +1119,58 @@ impl SourceCleanupDeleteBarrier {
|
||||
.expect("source cleanup should reach the pre-delete barrier");
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_fence_pending(&self) {
|
||||
tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified())
|
||||
.await
|
||||
.expect("source cleanup should attempt the fixed mutation fence");
|
||||
}
|
||||
|
||||
pub(crate) fn is_paused(&self) -> bool {
|
||||
self.state.is_paused.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) {
|
||||
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison")
|
||||
.iter()
|
||||
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.fence_pending.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for SourceCleanupDeleteBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
barriers.retain(|state| !Arc::ptr_eq(state, &self.state));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) {
|
||||
let barrier = SOURCE_CLEANUP_DELETE_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
let barrier = SOURCE_CLEANUP_DELETE_BARRIERS
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("source cleanup delete barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.iter()
|
||||
.find(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.is_paused.store(true, Ordering::Release);
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
@@ -1154,11 +1186,20 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
op_label: &str,
|
||||
) -> std::result::Result<ObjectInfo, SourceCleanupError> {
|
||||
let cleanup_key = encode_dir_object(object);
|
||||
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
|
||||
let _guard = ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(Error::from)?;
|
||||
let source_guard = if bucket_fence
|
||||
.object_mutation_fence
|
||||
.is_some_and(SourceCleanupMutationFence::source_lock_covered)
|
||||
{
|
||||
None
|
||||
} else {
|
||||
let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?;
|
||||
Some(
|
||||
ns_lock
|
||||
.get_write_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(Error::from)?,
|
||||
)
|
||||
};
|
||||
|
||||
if bucket_fence
|
||||
.lifecycle_guard
|
||||
@@ -1168,6 +1209,14 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
"{op_label}: bucket incarnation fence was lost before source cleanup"
|
||||
))));
|
||||
}
|
||||
if bucket_fence
|
||||
.object_mutation_fence
|
||||
.is_some_and(SourceCleanupMutationFence::is_lock_lost)
|
||||
{
|
||||
return Err(SourceCleanupError::Storage(Error::other(format!(
|
||||
"{op_label}: object mutation fence was lost before source cleanup"
|
||||
))));
|
||||
}
|
||||
|
||||
ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?;
|
||||
|
||||
@@ -1182,7 +1231,12 @@ pub(crate) async fn cleanup_source_entry_if_unchanged(
|
||||
expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
opts.add_namespace_lock_guard(&_guard);
|
||||
if let Some(source_guard) = source_guard.as_ref() {
|
||||
opts.add_namespace_lock_guard(source_guard);
|
||||
}
|
||||
if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence {
|
||||
object_mutation_fence.add_namespace_lock_fence(&mut opts);
|
||||
}
|
||||
if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard {
|
||||
opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard);
|
||||
}
|
||||
@@ -1330,6 +1384,37 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn migrate_decommission_object(
|
||||
store: Arc<ECStore>,
|
||||
pool_idx: usize,
|
||||
bucket: String,
|
||||
rd: GetObjectReader,
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
) -> Result<()> {
|
||||
let source = rd.object_info.clone();
|
||||
let _mutation_fence = store
|
||||
.acquire_decommission_object_mutation_fence(&bucket, &source.name)
|
||||
.await?;
|
||||
let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source)
|
||||
.await?
|
||||
.ok_or(Error::FileNotFound)?;
|
||||
if !is_equivalent_data_movement_object_identity(&source, ¤t, true, false) {
|
||||
return Err(Error::FileNotFound);
|
||||
}
|
||||
|
||||
migrate_object_inner(
|
||||
store,
|
||||
pool_idx,
|
||||
bucket,
|
||||
rd,
|
||||
source_bucket_incarnation_id,
|
||||
op_label,
|
||||
Some(&_mutation_fence),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn migrate_object(
|
||||
store: Arc<ECStore>,
|
||||
pool_idx: usize,
|
||||
@@ -1337,6 +1422,18 @@ pub(crate) async fn migrate_object(
|
||||
rd: GetObjectReader,
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
) -> Result<()> {
|
||||
migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await
|
||||
}
|
||||
|
||||
async fn migrate_object_inner(
|
||||
store: Arc<ECStore>,
|
||||
pool_idx: usize,
|
||||
bucket: String,
|
||||
rd: GetObjectReader,
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<()> {
|
||||
let object_info = rd.object_info.clone();
|
||||
let has_part_checksums = object_info
|
||||
@@ -1350,7 +1447,7 @@ pub(crate) async fn migrate_object(
|
||||
let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
|
||||
new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
|
||||
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts)
|
||||
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
|
||||
.await
|
||||
{
|
||||
Ok(res) => res,
|
||||
@@ -1448,7 +1545,7 @@ pub(crate) async fn migrate_object(
|
||||
if let Err(err) = store
|
||||
.clone()
|
||||
.complete_multipart_upload_for_data_movement(
|
||||
target_pool_idx,
|
||||
(target_pool_idx, mutation_fence),
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&res.upload_id,
|
||||
@@ -1609,7 +1706,7 @@ pub(crate) async fn migrate_object(
|
||||
let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
|
||||
put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
let (target_pool_idx, put_result) = store
|
||||
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts)
|
||||
.put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence)
|
||||
.await
|
||||
.map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?;
|
||||
if let Err(err) = put_result {
|
||||
@@ -3541,25 +3638,47 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_precondition_conflict_rejects_newer_delete_marker() {
|
||||
let source = ObjectInfo {
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
delete_marker: true,
|
||||
etag: None,
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
fn test_precondition_conflict_accepts_only_newer_null_delete_marker() {
|
||||
for version_id in [None, Some(Uuid::nil())] {
|
||||
let source = ObjectInfo {
|
||||
version_id,
|
||||
size: 128,
|
||||
etag: Some("etag-source".to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
};
|
||||
let target = ObjectInfo {
|
||||
delete_marker: true,
|
||||
etag: None,
|
||||
mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND),
|
||||
..source.clone()
|
||||
};
|
||||
|
||||
let should_resume =
|
||||
resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1)
|
||||
.expect("delete marker conflict should be evaluated");
|
||||
assert!(
|
||||
resolve_data_movement_overwrite_resume_result(
|
||||
&Error::PreconditionFailed,
|
||||
Ok(Some(target.clone())),
|
||||
&source,
|
||||
0,
|
||||
1,
|
||||
)
|
||||
.expect("newer null delete marker should be evaluated")
|
||||
);
|
||||
|
||||
assert!(!should_resume);
|
||||
let mut same_time = target.clone();
|
||||
same_time.mod_time = source.mod_time;
|
||||
assert!(
|
||||
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,)
|
||||
.expect("same-generation null delete marker should be rejected")
|
||||
);
|
||||
|
||||
let mut versioned = target;
|
||||
versioned.version_id = Some(Uuid::new_v4());
|
||||
assert!(
|
||||
!resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,)
|
||||
.expect("a UUID delete marker must not erase a null source version")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1391,7 +1391,37 @@ impl BucketUsageAccumulator {
|
||||
}
|
||||
|
||||
pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
|
||||
let logical_size = u64::try_from(object.get_actual_size().map_err(Error::other)?).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
// A compressed object may carry -1 while the transformed size is unknown
|
||||
// (legacy streaming sentinel). In that case the persisted physical size
|
||||
// is still a valid accounting floor; every other negative value is corrupt.
|
||||
// An explicit negative `actual-size` metadata value is corrupt, however:
|
||||
// the sentinel is only valid in the in-memory/object-part field written by
|
||||
// the legacy streaming path, not as a persisted declared size.
|
||||
let compressed = object.is_compressed();
|
||||
if object.actual_size < -1 || (object.actual_size == -1 && !compressed) {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
if object
|
||||
.parts
|
||||
.iter()
|
||||
.any(|part| part.actual_size < -1 || (part.actual_size < 0 && !compressed))
|
||||
{
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
let declared_actual_size = rustfs_utils::http::get_str(&object.user_defined, rustfs_utils::http::SUFFIX_ACTUAL_SIZE)
|
||||
.filter(|value| !value.is_empty());
|
||||
if declared_actual_size
|
||||
.as_deref()
|
||||
.and_then(|value| value.parse::<i64>().ok())
|
||||
.is_some_and(|size| size < 0)
|
||||
{
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
let logical_size = match object.get_actual_size().map_err(Error::other)? {
|
||||
size if size == -1 && compressed && declared_actual_size.is_none() => None,
|
||||
size if size >= 0 => Some(u64::try_from(size).map_err(|_| Error::PartMissingOrCorrupt)?),
|
||||
_ => return Err(Error::PartMissingOrCorrupt),
|
||||
};
|
||||
let persisted_part_size = if object.parts.is_empty() {
|
||||
u64::try_from(object.size).map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
} else {
|
||||
@@ -1399,12 +1429,8 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
|
||||
// Compressed streaming objects persist -1 when the transformed
|
||||
// part size is unknown. The physical part size remains a valid
|
||||
// quota floor; reject only non-negative values that overflow.
|
||||
let actual_size = if part.actual_size < 0 {
|
||||
if object.is_compressed() {
|
||||
0
|
||||
} else {
|
||||
return Err(Error::PartMissingOrCorrupt);
|
||||
}
|
||||
let actual_size = if part.actual_size == -1 {
|
||||
0
|
||||
} else {
|
||||
u64::try_from(part.actual_size).map_err(|_| Error::PartMissingOrCorrupt)?
|
||||
};
|
||||
@@ -1412,7 +1438,7 @@ pub fn quota_object_size(object: &ObjectInfo) -> Result<u64, Error> {
|
||||
total.checked_add(part_size).ok_or(Error::PartMissingOrCorrupt)
|
||||
})?
|
||||
};
|
||||
Ok(logical_size.max(persisted_part_size))
|
||||
Ok(logical_size.unwrap_or(0).max(persisted_part_size))
|
||||
}
|
||||
|
||||
type UsageVersionPage = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
@@ -3320,6 +3346,80 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_object_size_accepts_compressed_unknown_actual_size_sentinel() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
let object = ObjectInfo {
|
||||
size: 400,
|
||||
actual_size: -1,
|
||||
user_defined: Arc::new(metadata),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(quota_object_size(&object).expect("compressed sentinel is valid"), 400);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_object_size_rejects_compressed_part_sum_overflow() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
let object = ObjectInfo {
|
||||
size: 1,
|
||||
user_defined: Arc::new(metadata),
|
||||
parts: Arc::new(vec![
|
||||
rustfs_filemeta::ObjectPartInfo {
|
||||
actual_size: i64::MAX,
|
||||
..Default::default()
|
||||
},
|
||||
rustfs_filemeta::ObjectPartInfo {
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
},
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(matches!(quota_object_size(&object), Err(Error::Io(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_object_size_rejects_negative_values_other_than_the_compressed_sentinel() {
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
let corrupt_object = ObjectInfo {
|
||||
size: 400,
|
||||
actual_size: -2,
|
||||
user_defined: Arc::new(metadata.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(quota_object_size(&corrupt_object), Err(Error::PartMissingOrCorrupt)));
|
||||
|
||||
let corrupt_part = ObjectInfo {
|
||||
size: 400,
|
||||
user_defined: Arc::new(metadata),
|
||||
parts: Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
size: 400,
|
||||
actual_size: -2,
|
||||
..Default::default()
|
||||
}]),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(matches!(quota_object_size(&corrupt_part), Err(Error::PartMissingOrCorrupt)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn live_bucket_usage_refreshes_are_coalesced_only_while_in_flight() {
|
||||
|
||||
@@ -858,6 +858,7 @@ const EVENT_DISK_LOCAL_DIRECT_IO_FALLBACK: &str = "disk_local_direct_io_fallback
|
||||
#[cfg(target_os = "linux")]
|
||||
const EVENT_DISK_LOCAL_URING_LATCH_OFF: &str = "disk_local_uring_latch_off";
|
||||
const EVENT_DISK_LOCAL_DELETE_FAILED: &str = "disk_local_delete_failed";
|
||||
const EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED: &str = "disk_local_delete_rollback_failed";
|
||||
const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts";
|
||||
const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed";
|
||||
const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed";
|
||||
@@ -6106,6 +6107,43 @@ impl LocalDisk {
|
||||
Ok((bytes, modtime))
|
||||
}
|
||||
|
||||
async fn write_missing_delete_marker(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
fi: FileInfo,
|
||||
object_dir: &Path,
|
||||
xl_path: &Path,
|
||||
rollback_dir: Option<Uuid>,
|
||||
) -> Result<()> {
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
let rollback_path = object_dir.join(rollback_dir.to_string());
|
||||
fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
|
||||
fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
if let Err(err) = self.write_metadata("", volume, path, fi).await {
|
||||
if let Some(rollback_dir) = rollback_dir
|
||||
&& let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
result = "failed",
|
||||
volume,
|
||||
path,
|
||||
rollback_dir = %rollback_dir,
|
||||
error = ?restore_err,
|
||||
"Disk local delete rollback failed"
|
||||
);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> {
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?;
|
||||
@@ -6123,7 +6161,20 @@ impl LocalDisk {
|
||||
return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await;
|
||||
}
|
||||
|
||||
let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?;
|
||||
let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await {
|
||||
Ok(data) => data,
|
||||
Err(DiskError::FileNotFound) => {
|
||||
// `deleted` alone can be an explicit marker purge; only
|
||||
// `mark_deleted` may create metadata that was not present.
|
||||
let Some(delete_marker) = fis.iter().find(|fi| fi.deleted && fi.mark_deleted).cloned() else {
|
||||
return Err(DiskError::FileNotFound);
|
||||
};
|
||||
return self
|
||||
.write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir)
|
||||
.await;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
if data.is_empty() {
|
||||
return Err(DiskError::FileNotFound);
|
||||
@@ -10422,29 +10473,9 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
if fi.deleted && force_del_marker {
|
||||
if let Some(rollback_dir) = rollback_dir {
|
||||
let rollback_path = file_path.join(rollback_dir.to_string());
|
||||
fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?;
|
||||
fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), [])
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
if let Err(err) = self.write_metadata("", volume, path, fi).await {
|
||||
if let Some(rollback_dir) = rollback_dir
|
||||
&& let Err(restore_err) =
|
||||
restore_delete_rollback(file_path.as_path(), &xl_path, rollback_dir, &self.publication_root).await
|
||||
{
|
||||
warn!(
|
||||
volume,
|
||||
path,
|
||||
rollback_dir = %rollback_dir,
|
||||
error = ?restore_err,
|
||||
"failed to restore metadata after delete marker commit error"
|
||||
);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
return Ok(());
|
||||
return self
|
||||
.write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir)
|
||||
.await;
|
||||
}
|
||||
|
||||
return if fi.version_id.is_some() {
|
||||
|
||||
@@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_DISK: &str = "disk";
|
||||
const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified";
|
||||
const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM";
|
||||
const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4;
|
||||
|
||||
pub fn part_transaction_path(part_path: &str) -> String {
|
||||
match part_path.rsplit_once('/') {
|
||||
@@ -62,6 +64,7 @@ use bytes::Bytes;
|
||||
use endpoint::Endpoint;
|
||||
use error::DiskError;
|
||||
use error::{Error, Result};
|
||||
use futures::stream::{self, StreamExt};
|
||||
use local::LocalDisk;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
use rustfs_madmin::info_commands::DiskMetrics;
|
||||
@@ -417,6 +420,14 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result<Vec<BatchReadVersionResp>> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.batch_read_version(req).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
|
||||
match self {
|
||||
@@ -1028,36 +1039,47 @@ where
|
||||
D: DiskAPI + ?Sized,
|
||||
{
|
||||
validate_batch_read_version_item_count(req.items.len())?;
|
||||
let parallelism = batch_read_version_server_parallelism();
|
||||
|
||||
let mut responses = Vec::with_capacity(req.items.len());
|
||||
for (index, item) in req.items.iter().enumerate() {
|
||||
let response = match disk
|
||||
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
|
||||
.await
|
||||
{
|
||||
Ok(file_info) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path.clone(),
|
||||
version_id: item.version_id.clone(),
|
||||
success: true,
|
||||
file_info,
|
||||
error: String::new(),
|
||||
},
|
||||
Err(err) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path.clone(),
|
||||
version_id: item.version_id.clone(),
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: err.to_string(),
|
||||
},
|
||||
};
|
||||
responses.push(response);
|
||||
}
|
||||
let mut responses = stream::iter(req.items.into_iter().enumerate())
|
||||
.map(|(index, item)| async move {
|
||||
match disk
|
||||
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
|
||||
.await
|
||||
{
|
||||
Ok(file_info) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path,
|
||||
version_id: item.version_id,
|
||||
success: true,
|
||||
file_info,
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
},
|
||||
Err(err) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path,
|
||||
version_id: item.version_id,
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: err.to_string(),
|
||||
error_code: err.to_u32(),
|
||||
},
|
||||
}
|
||||
})
|
||||
.buffer_unordered(parallelism)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
responses.sort_unstable_by_key(|response| response.index);
|
||||
|
||||
Ok(responses)
|
||||
}
|
||||
|
||||
fn batch_read_version_server_parallelism() -> usize {
|
||||
rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM)
|
||||
.clamp(1, BATCH_READ_VERSION_MAX_ITEMS)
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct CheckPartsResp {
|
||||
pub results: Vec<usize>,
|
||||
@@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp {
|
||||
pub success: bool,
|
||||
pub file_info: FileInfo,
|
||||
pub error: String,
|
||||
#[serde(default)]
|
||||
pub error_code: u32,
|
||||
}
|
||||
|
||||
pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> {
|
||||
@@ -1417,6 +1441,26 @@ mod tests {
|
||||
assert!(!partial_valid_location.valid());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_server_parallelism_defaults_to_conservative_four() {
|
||||
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || {
|
||||
assert_eq!(batch_read_version_server_parallelism(), 4);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_server_parallelism_honors_env_with_bounds() {
|
||||
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || {
|
||||
assert_eq!(batch_read_version_server_parallelism(), 8);
|
||||
});
|
||||
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || {
|
||||
assert_eq!(batch_read_version_server_parallelism(), 1);
|
||||
});
|
||||
temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || {
|
||||
assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS);
|
||||
});
|
||||
}
|
||||
|
||||
/// Test FileInfoVersions find_version_index
|
||||
#[test]
|
||||
fn test_file_info_versions_find_version_index() {
|
||||
|
||||
@@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() {
|
||||
cluster::rpc::shutdown_background_monitors();
|
||||
}
|
||||
|
||||
/// Publish that the process is ready to serve user-object GET traffic.
|
||||
///
|
||||
/// Experimental metadata coalescing is allowed to run only after this point so
|
||||
/// startup and internal metadata reads keep the original per-disk path.
|
||||
pub fn mark_get_metadata_read_version_coalescing_service_ready() {
|
||||
runtime::global::mark_get_metadata_read_version_coalescing_service_ready();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod rio_tests {
|
||||
#[test]
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::storage_api_contracts::{
|
||||
pub struct NamespaceLockFence {
|
||||
signals: Arc<Vec<Arc<rustfs_lock::distributed_lock::LockLostSignal>>>,
|
||||
#[cfg(test)]
|
||||
forced_lost: Arc<std::sync::atomic::AtomicBool>,
|
||||
forced_lost: Arc<Vec<Arc<std::sync::atomic::AtomicBool>>>,
|
||||
}
|
||||
|
||||
impl Debug for NamespaceLockFence {
|
||||
@@ -40,13 +40,17 @@ impl NamespaceLockFence {
|
||||
Self {
|
||||
signals: Arc::default(),
|
||||
#[cfg(test)]
|
||||
forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_lock_lost(&self) -> bool {
|
||||
#[cfg(test)]
|
||||
if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
|
||||
if self
|
||||
.forced_lost
|
||||
.iter()
|
||||
.any(|lost| lost.load(std::sync::atomic::Ordering::Acquire))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
self.signals.iter().any(|signal| signal.is_lost())
|
||||
@@ -57,27 +61,26 @@ impl NamespaceLockFence {
|
||||
}
|
||||
|
||||
fn extend(&mut self, other: &Self) {
|
||||
if Arc::ptr_eq(&self.signals, &other.signals) {
|
||||
return;
|
||||
if !Arc::ptr_eq(&self.signals, &other.signals) {
|
||||
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
|
||||
}
|
||||
Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned());
|
||||
#[cfg(test)]
|
||||
if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) {
|
||||
self.forced_lost.store(true, std::sync::atomic::Ordering::Release);
|
||||
if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) {
|
||||
Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn lost_for_test() -> Self {
|
||||
let fence = Self::new();
|
||||
fence.forced_lost.store(true, std::sync::atomic::Ordering::Release);
|
||||
fence.forced_lost[0].store(true, std::sync::atomic::Ordering::Release);
|
||||
fence
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn loss_handle_for_test() -> (Self, Arc<std::sync::atomic::AtomicBool>) {
|
||||
let fence = Self::new();
|
||||
(fence.clone(), Arc::clone(&fence.forced_lost))
|
||||
(fence.clone(), Arc::clone(&fence.forced_lost[0]))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -411,6 +414,13 @@ impl ObjectOptions {
|
||||
self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) {
|
||||
self.namespace_lock_fence
|
||||
.get_or_insert_with(NamespaceLockFence::new)
|
||||
.extend(fence);
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) {
|
||||
self.lifecycle_delete_all_journal
|
||||
.get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default())));
|
||||
@@ -689,6 +699,9 @@ impl ObjectInfo {
|
||||
}
|
||||
|
||||
pub fn get_actual_size(&self) -> std::io::Result<i64> {
|
||||
if self.actual_size < -1 || (self.actual_size == -1 && !self.is_compressed()) {
|
||||
return Err(std::io::Error::other("invalid negative actual size"));
|
||||
}
|
||||
if self.actual_size > 0 {
|
||||
return Ok(self.actual_size);
|
||||
}
|
||||
@@ -700,10 +713,25 @@ impl ObjectInfo {
|
||||
let size = size_str.parse::<i64>().map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
return Ok(size);
|
||||
}
|
||||
let mut actual_size = 0;
|
||||
self.parts.iter().for_each(|part| {
|
||||
actual_size += part.actual_size;
|
||||
});
|
||||
if self.actual_size == -1 && self.parts.is_empty() {
|
||||
return Ok(-1);
|
||||
}
|
||||
let mut actual_size = 0_i64;
|
||||
let mut unknown = false;
|
||||
for part in self.parts.iter() {
|
||||
match part.actual_size {
|
||||
-1 => unknown = true,
|
||||
size if size >= 0 => {
|
||||
actual_size = actual_size
|
||||
.checked_add(size)
|
||||
.ok_or_else(|| std::io::Error::other("compressed actual size overflow"))?;
|
||||
}
|
||||
_ => return Err(std::io::Error::other("invalid negative compressed part size")),
|
||||
}
|
||||
}
|
||||
if unknown {
|
||||
return Ok(-1);
|
||||
}
|
||||
if actual_size == 0 && actual_size != self.size {
|
||||
return Err(std::io::Error::other(format!("invalid decompressed size {} {}", actual_size, self.size)));
|
||||
}
|
||||
@@ -718,6 +746,18 @@ impl ObjectInfo {
|
||||
Ok(self.size)
|
||||
}
|
||||
|
||||
/// Returns a non-negative size for client and replication boundaries.
|
||||
///
|
||||
/// Compressed legacy metadata can retain the internal `-1` unknown-size
|
||||
/// sentinel. Those boundaries cannot emit a negative length, so they use
|
||||
/// the persisted physical size while quota accounting keeps the sentinel
|
||||
/// distinction in [`crate::data_usage::quota_object_size`].
|
||||
pub fn get_actual_size_or_physical(&self) -> i64 {
|
||||
self.get_actual_size()
|
||||
.map(|size| if size >= 0 { size } else { self.size.max(0) })
|
||||
.unwrap_or_else(|_| self.size.max(0))
|
||||
}
|
||||
|
||||
pub fn from_file_info(fi: &FileInfo, bucket: &str, object: &str, versioned: bool) -> ObjectInfo {
|
||||
let mut version_id = fi.version_id;
|
||||
|
||||
|
||||
@@ -25,7 +25,10 @@ use lazy_static::lazy_static;
|
||||
use rustfs_lock::client::LockClient;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, OnceLock},
|
||||
sync::{
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::SystemTime,
|
||||
};
|
||||
use tokio::sync::{OnceCell, RwLock};
|
||||
@@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000;
|
||||
pub const DISK_FILL_FRACTION: f64 = 0.99;
|
||||
pub const DISK_RESERVE_FRACTION: f64 = 0.15;
|
||||
|
||||
static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() {
|
||||
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool {
|
||||
GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
// Global singletons for backward compatibility with MinIO port.
|
||||
// These should be migrated to AppContext over time.
|
||||
// See issue #730 for migration plan.
|
||||
|
||||
@@ -160,6 +160,10 @@ pub struct InstanceContext {
|
||||
/// workers (scanner/heal/tier/lifecycle) without touching another instance.
|
||||
/// Replaces the process-global cancel-token static.
|
||||
background_cancel_token: OnceLock<CancellationToken>,
|
||||
/// Serializes decommission data-movement operations with cancellation and
|
||||
/// a subsequent restart. Readers are held across one object side effect;
|
||||
/// the transition path takes the writer after cancelling the routine.
|
||||
decommission_operation_gate: Arc<RwLock<()>>,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
@@ -200,6 +204,7 @@ impl InstanceContext {
|
||||
local_disk_set_drives: Arc::new(RwLock::new(Vec::new())),
|
||||
bucket_metadata_sys: std::sync::Mutex::new(None),
|
||||
background_cancel_token: OnceLock::new(),
|
||||
decommission_operation_gate: Arc::new(RwLock::new(())),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
@@ -218,6 +223,10 @@ impl InstanceContext {
|
||||
self.lock_manager.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn decommission_operation_gate(&self) -> Arc<RwLock<()>> {
|
||||
Arc::clone(&self.decommission_operation_gate)
|
||||
}
|
||||
|
||||
/// Install the application-owned object-encryption resolver once.
|
||||
pub fn set_object_encryption_resolver(
|
||||
&self,
|
||||
|
||||
@@ -256,6 +256,10 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
|
||||
cycle_max_duration_seconds: metrics.cycle_max_duration_seconds,
|
||||
cycle_max_objects: metrics.cycle_max_objects,
|
||||
cycle_max_directories: metrics.cycle_max_directories,
|
||||
cycle_timeout_total: metrics.cycle_timeout_total,
|
||||
cycle_recovery_required_total: metrics.cycle_recovery_required_total,
|
||||
cycle_last_progress_age: metrics.cycle_last_progress_age,
|
||||
leader_lease_without_progress: metrics.leader_lease_without_progress,
|
||||
bitrot_cycle_enabled: metrics.bitrot_cycle_enabled,
|
||||
bitrot_cycle_seconds: metrics.bitrot_cycle_seconds,
|
||||
scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport {
|
||||
@@ -611,6 +615,10 @@ mod test {
|
||||
current_started: chrono_to_jiff_timestamp(current_started),
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
cycle_timeout_total: 3,
|
||||
cycle_recovery_required_total: 2,
|
||||
cycle_last_progress_age: 17,
|
||||
leader_lease_without_progress: true,
|
||||
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
|
||||
source: "usage".to_string(),
|
||||
cycles: 2,
|
||||
@@ -622,6 +630,10 @@ mod test {
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
assert_eq!(scanner.cycle_timeout_total, 3);
|
||||
assert_eq!(scanner.cycle_recovery_required_total, 2);
|
||||
assert_eq!(scanner.cycle_last_progress_age, 17);
|
||||
assert!(scanner.leader_lease_without_progress);
|
||||
let usage = scanner
|
||||
.partial_cycles_by_source
|
||||
.iter()
|
||||
|
||||
@@ -334,6 +334,7 @@ impl ECStore {
|
||||
lifecycle_guard: bucket_incarnation_fence
|
||||
.as_ref()
|
||||
.and_then(|guard| guard.namespace_lock_guard()),
|
||||
..Default::default()
|
||||
},
|
||||
"rebalance",
|
||||
),
|
||||
|
||||
@@ -53,11 +53,12 @@ use crate::diagnostics::get::{
|
||||
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
|
||||
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
|
||||
};
|
||||
use crate::disk::disk_store::DiskStoreRenameDataExt;
|
||||
use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout};
|
||||
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
|
||||
use crate::disk::{
|
||||
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
|
||||
PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk,
|
||||
OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction,
|
||||
STORAGE_FORMAT_FILE_BACKUP, part_transaction_path,
|
||||
};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::io_support::bitrot::ShardReader;
|
||||
@@ -75,7 +76,7 @@ use std::{
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
sync::{
|
||||
OnceLock,
|
||||
Arc, OnceLock,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
task::{Context, Poll},
|
||||
@@ -94,6 +95,242 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String {
|
||||
[bucket, object].join("/")
|
||||
}
|
||||
|
||||
fn read_version_coalescing_enabled() -> bool {
|
||||
let enabled = || {
|
||||
rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE)
|
||||
.is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on"))
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
{
|
||||
enabled()
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
static ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
*ENABLED.get_or_init(enabled)
|
||||
}
|
||||
}
|
||||
|
||||
fn read_version_coalescing_delay() -> Duration {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let micros = rustfs_utils::get_env_u64(
|
||||
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
|
||||
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
|
||||
);
|
||||
Duration::from_micros(micros)
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
static DELAY: OnceLock<Duration> = OnceLock::new();
|
||||
*DELAY.get_or_init(|| {
|
||||
Duration::from_micros(rustfs_utils::get_env_u64(
|
||||
ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
|
||||
DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS,
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct CoalescedReadVersionRequest {
|
||||
item: BatchReadVersionItem,
|
||||
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
struct ReadVersionCoalescerKey {
|
||||
disk: usize,
|
||||
incl_free_versions: bool,
|
||||
read_data: bool,
|
||||
healing: bool,
|
||||
}
|
||||
|
||||
impl ReadVersionCoalescerKey {
|
||||
fn new(disk: &DiskStore, opts: &ReadOptions) -> Self {
|
||||
Self {
|
||||
disk: Arc::as_ptr(disk) as usize,
|
||||
incl_free_versions: opts.incl_free_versions,
|
||||
read_data: opts.read_data,
|
||||
healing: opts.healing,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct ReadVersionCoalescer {
|
||||
lanes: HashMap<ReadVersionCoalescerKey, Vec<CoalescedReadVersionRequest>>,
|
||||
}
|
||||
|
||||
fn read_version_coalescer() -> &'static Mutex<ReadVersionCoalescer> {
|
||||
static COALESCER: OnceLock<Mutex<ReadVersionCoalescer>> = OnceLock::new();
|
||||
COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default()))
|
||||
}
|
||||
|
||||
fn record_read_version_coalescer_event(event: &'static str, item_count: usize) {
|
||||
counter!(
|
||||
METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL,
|
||||
"event" => event,
|
||||
"item_count" => item_count.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
async fn read_version_via_coalescer(
|
||||
disk: DiskStore,
|
||||
org_bucket: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
if !allow_coalescing || !read_version_coalescing_enabled() {
|
||||
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
|
||||
}
|
||||
if !matches!(disk.as_ref(), Disk::Remote(_)) {
|
||||
record_read_version_coalescer_event("bypass_non_remote", 1);
|
||||
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
|
||||
}
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let item = BatchReadVersionItem {
|
||||
org_volume: org_bucket.to_string(),
|
||||
volume: bucket.to_string(),
|
||||
path: object.to_string(),
|
||||
version_id: version_id.to_string(),
|
||||
};
|
||||
let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
|
||||
let pending = {
|
||||
let mut coalescer = read_version_coalescer().lock().await;
|
||||
let lane = coalescer.lanes.entry(lane_key).or_default();
|
||||
let schedule_delayed_flush = lane.is_empty();
|
||||
lane.push(CoalescedReadVersionRequest { item, tx });
|
||||
if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
|
||||
coalescer.lanes.remove(&lane_key)
|
||||
} else if schedule_delayed_flush {
|
||||
let disk = disk.clone();
|
||||
let task_opts = *opts;
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(read_version_coalescing_delay()).await;
|
||||
flush_read_version_coalescer_lane(lane_key, disk, task_opts).await;
|
||||
});
|
||||
None
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(pending) = pending {
|
||||
flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await;
|
||||
}
|
||||
|
||||
rx.await
|
||||
.unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed")))
|
||||
}
|
||||
|
||||
async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) {
|
||||
let pending = {
|
||||
let mut coalescer = read_version_coalescer().lock().await;
|
||||
coalescer.lanes.remove(&lane_key).unwrap_or_default()
|
||||
};
|
||||
flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await;
|
||||
}
|
||||
|
||||
async fn flush_read_version_coalescer_pending(
|
||||
lane_key: ReadVersionCoalescerKey,
|
||||
disk: DiskStore,
|
||||
opts: ReadOptions,
|
||||
pending: Vec<CoalescedReadVersionRequest>,
|
||||
) {
|
||||
if pending.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
{
|
||||
let mut observed_paths = HashSet::new();
|
||||
for request in &pending {
|
||||
if observed_paths.insert(request.item.path.as_str()) {
|
||||
disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut senders = Vec::with_capacity(pending.len());
|
||||
let mut items = Vec::with_capacity(pending.len());
|
||||
for request in pending {
|
||||
senders.push(request.tx);
|
||||
items.push(request.item);
|
||||
}
|
||||
|
||||
let expected_items = items.clone();
|
||||
record_read_version_coalescer_event("attempted_batch", items.len());
|
||||
let result =
|
||||
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
};
|
||||
match result {
|
||||
Ok(responses) => {
|
||||
let results = map_batch_read_version_responses(&expected_items, responses);
|
||||
for (tx, result) in senders.into_iter().zip(results) {
|
||||
let _ = tx.send(result);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let message = err.to_string();
|
||||
for tx in senders {
|
||||
let _ = tx.send(Err(DiskError::other(message.clone())));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn map_batch_read_version_responses(
|
||||
expected_items: &[BatchReadVersionItem],
|
||||
responses: Vec<BatchReadVersionResp>,
|
||||
) -> Vec<crate::disk::error::Result<FileInfo>> {
|
||||
let mut results = (0..expected_items.len())
|
||||
.map(|_| Err(DiskError::other("coalesced read_version response missing")))
|
||||
.collect::<Vec<_>>();
|
||||
let mut seen = vec![false; expected_items.len()];
|
||||
for response in responses {
|
||||
let Some(expected) = expected_items.get(response.index) else {
|
||||
continue;
|
||||
};
|
||||
let Some(slot) = results.get_mut(response.index) else {
|
||||
continue;
|
||||
};
|
||||
if seen[response.index] {
|
||||
*slot = Err(DiskError::other("coalesced read_version response duplicate index"));
|
||||
continue;
|
||||
}
|
||||
seen[response.index] = true;
|
||||
if response.path != expected.path || response.version_id != expected.version_id {
|
||||
*slot = Err(DiskError::other("coalesced read_version response identity mismatch"));
|
||||
} else {
|
||||
*slot = if response.success {
|
||||
Ok(response.file_info)
|
||||
} else {
|
||||
Err(batch_read_version_response_error(response.error_code, response.error))
|
||||
};
|
||||
}
|
||||
}
|
||||
results
|
||||
}
|
||||
|
||||
fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError {
|
||||
match DiskError::from_u32(error_code) {
|
||||
Some(DiskError::Io(_)) | None => DiskError::other(error),
|
||||
Some(error) => error,
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn bounded_metadata_fanout_order(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -133,11 +370,15 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order(
|
||||
order
|
||||
}
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::sync::{Mutex, RwLock, oneshot};
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
|
||||
pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
|
||||
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
|
||||
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
|
||||
const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
|
||||
const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
|
||||
pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
|
||||
/// Default reader-setup strategy for the GET read path (rustfs/backlog#1215,
|
||||
/// #1159, #923).
|
||||
@@ -2356,6 +2597,7 @@ impl SetDisks {
|
||||
false,
|
||||
true,
|
||||
0,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
Ok((ress, errors))
|
||||
@@ -2386,6 +2628,36 @@ impl SetDisks {
|
||||
true,
|
||||
caller_allows_early_stop,
|
||||
default_parity_count,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object(
|
||||
disks: &[Option<DiskStore>],
|
||||
org_bucket: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: &str,
|
||||
read_data: bool,
|
||||
incl_free_versions: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
default_parity_count: usize,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
|
||||
Self::read_all_fileinfo_inner(
|
||||
disks,
|
||||
org_bucket,
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
read_data,
|
||||
false,
|
||||
incl_free_versions,
|
||||
true,
|
||||
caller_allows_early_stop,
|
||||
default_parity_count,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2408,6 +2680,7 @@ impl SetDisks {
|
||||
// subset would fail write quorum (backlog#872 regression).
|
||||
caller_allows_early_stop: bool,
|
||||
default_parity_count: usize,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
|
||||
let early_stop_enabled =
|
||||
caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled());
|
||||
@@ -2424,6 +2697,7 @@ impl SetDisks {
|
||||
healing,
|
||||
incl_free_versions,
|
||||
default_parity_count,
|
||||
allow_coalescing,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -2446,6 +2720,7 @@ impl SetDisks {
|
||||
healing,
|
||||
incl_free_versions,
|
||||
observe,
|
||||
allow_coalescing,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -2461,6 +2736,7 @@ impl SetDisks {
|
||||
healing: bool,
|
||||
incl_free_versions: bool,
|
||||
observe: bool,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
|
||||
let fanout_start = observe.then(Instant::now);
|
||||
let mut ress = Vec::with_capacity(disks.len());
|
||||
@@ -2492,7 +2768,7 @@ impl SetDisks {
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
@@ -2559,6 +2835,7 @@ impl SetDisks {
|
||||
healing: bool,
|
||||
incl_free_versions: bool,
|
||||
default_parity_count: usize,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
|
||||
let fanout_start = Instant::now();
|
||||
let mut ress = vec![FileInfo::default(); disks.len()];
|
||||
@@ -2607,7 +2884,7 @@ impl SetDisks {
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts)
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
@@ -5737,6 +6014,7 @@ pub(crate) mod disk_call_counters {
|
||||
|
||||
/// Kind label for the per-disk `read_version` metadata RPC.
|
||||
pub const KIND_READ_VERSION: &str = "read_version";
|
||||
pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version";
|
||||
|
||||
/// Registry key: (object, kind, disk_index).
|
||||
type CountKey = (String, String, usize);
|
||||
@@ -6460,6 +6738,286 @@ mod tests {
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn metadata_read_version_coalescer_bypasses_local_disks() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "coalesced-read-version-local-bypass-bucket";
|
||||
let object_a = "coalesced-local-object-a";
|
||||
let object_b = "coalesced-local-object-b";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await;
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")),
|
||||
(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(object_a);
|
||||
let disks_a = disks.clone();
|
||||
let disks_b = disks.clone();
|
||||
let read_a = tokio::spawn(async move {
|
||||
SetDisks::read_all_fileinfo_observed_for_get_object(
|
||||
&disks_a, "", bucket, object_a, "", false, false, false, 2,
|
||||
)
|
||||
.await
|
||||
.map(|(file_infos, errors, _)| (file_infos, errors))
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
let read_b = tokio::spawn(async move {
|
||||
SetDisks::read_all_fileinfo_observed_for_get_object(
|
||||
&disks_b, "", bucket, object_b, "", false, false, false, 2,
|
||||
)
|
||||
.await
|
||||
.map(|(file_infos, errors, _)| (file_infos, errors))
|
||||
});
|
||||
|
||||
let (metadata_a, errs_a) = read_a
|
||||
.await
|
||||
.expect("first read task should not panic")
|
||||
.expect("first coalesced read should resolve");
|
||||
let (metadata_b, errs_b) = read_b
|
||||
.await
|
||||
.expect("second read task should not panic")
|
||||
.expect("second coalesced read should resolve");
|
||||
|
||||
assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS);
|
||||
assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS);
|
||||
assert!(errs_a.iter().all(Option::is_none));
|
||||
assert!(errs_b.iter().all(Option::is_none));
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
DISKS as u64,
|
||||
"local disks still execute the ordinary per-disk read_version path"
|
||||
);
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
|
||||
0,
|
||||
"GET coalescing targets internode RPC count only and must not batch local disk reads"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_read_version_coalescer_requires_get_object_intent() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "coalesced-read-version-default-bypass-bucket";
|
||||
let object = "default-bypass-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async {
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false)
|
||||
.await
|
||||
.expect("default metadata read should resolve");
|
||||
|
||||
assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64);
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_BATCH_READ_VERSION),
|
||||
0,
|
||||
"non-GET metadata paths must bypass coalescer even when the env gate is enabled"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_response_mapping_preserves_index_and_errors() {
|
||||
let expected_items = vec![
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
},
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-b".to_string(),
|
||||
version_id: "v-b".to_string(),
|
||||
},
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-c".to_string(),
|
||||
version_id: "v-c".to_string(),
|
||||
},
|
||||
];
|
||||
let ok_file_info = FileInfo {
|
||||
name: "object-a".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let responses = vec![
|
||||
BatchReadVersionResp {
|
||||
index: 2,
|
||||
path: "object-c".to_string(),
|
||||
version_id: "v-c".to_string(),
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: "disk read failed".to_string(),
|
||||
error_code: 0,
|
||||
},
|
||||
BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: true,
|
||||
file_info: ok_file_info,
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
},
|
||||
];
|
||||
|
||||
let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter();
|
||||
let first = results
|
||||
.next()
|
||||
.expect("slot 0 should exist")
|
||||
.expect("slot 0 should map the success response by index");
|
||||
assert_eq!(first.name, "object-a");
|
||||
|
||||
let missing = results
|
||||
.next()
|
||||
.expect("slot 1 should exist")
|
||||
.expect_err("slot 1 should stay missing");
|
||||
assert!(
|
||||
missing.to_string().contains("response missing"),
|
||||
"unexpected missing response error: {missing}"
|
||||
);
|
||||
|
||||
let failed = results
|
||||
.next()
|
||||
.expect("slot 2 should exist")
|
||||
.expect_err("slot 2 should map the response error");
|
||||
assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}");
|
||||
assert!(results.next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_response_mapping_preserves_typed_not_found_errors() {
|
||||
let expected_items = vec![
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
},
|
||||
BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-b".to_string(),
|
||||
version_id: "v-b".to_string(),
|
||||
},
|
||||
];
|
||||
let results = map_batch_read_version_responses(
|
||||
&expected_items,
|
||||
vec![
|
||||
BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: DiskError::FileNotFound.to_string(),
|
||||
error_code: DiskError::FileNotFound.to_u32(),
|
||||
},
|
||||
BatchReadVersionResp {
|
||||
index: 1,
|
||||
path: "object-b".to_string(),
|
||||
version_id: "v-b".to_string(),
|
||||
success: false,
|
||||
file_info: FileInfo::default(),
|
||||
error: DiskError::FileVersionNotFound.to_string(),
|
||||
error_code: DiskError::FileVersionNotFound.to_u32(),
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound)));
|
||||
assert!(matches!(
|
||||
results.get(1).expect("slot 1 should exist"),
|
||||
Err(DiskError::FileVersionNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() {
|
||||
let expected_items = vec![BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: "bucket".to_string(),
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
}];
|
||||
let mismatched = map_batch_read_version_responses(
|
||||
&expected_items,
|
||||
vec![BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-b".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: true,
|
||||
file_info: FileInfo {
|
||||
name: "object-b".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
}],
|
||||
)
|
||||
.pop()
|
||||
.expect("slot 0 should exist")
|
||||
.expect_err("identity mismatch should fail closed");
|
||||
assert!(
|
||||
mismatched.to_string().contains("identity mismatch"),
|
||||
"unexpected mismatch error: {mismatched}"
|
||||
);
|
||||
|
||||
let duplicate = map_batch_read_version_responses(
|
||||
&expected_items,
|
||||
vec![
|
||||
BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: true,
|
||||
file_info: FileInfo {
|
||||
name: "object-a".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
},
|
||||
BatchReadVersionResp {
|
||||
index: 0,
|
||||
path: "object-a".to_string(),
|
||||
version_id: "v-a".to_string(),
|
||||
success: true,
|
||||
file_info: FileInfo {
|
||||
name: "object-a".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
error: String::new(),
|
||||
error_code: 0,
|
||||
},
|
||||
],
|
||||
)
|
||||
.pop()
|
||||
.expect("slot 0 should exist")
|
||||
.expect_err("duplicate response index should fail closed");
|
||||
assert!(
|
||||
duplicate.to_string().contains("duplicate index"),
|
||||
"unexpected duplicate error: {duplicate}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Isolation guard: unobserved objects record nothing (so parallel tests do
|
||||
/// not inflate one another), and a scope clears its own counts on drop.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -97,7 +97,7 @@ use crate::storage_api_contracts::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations as _, MultipartUploadResult, PartInfo,
|
||||
},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
|
||||
object::{DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO as _, ObjectOperations as _, ObjectToDelete},
|
||||
range::HTTPRangeSpec,
|
||||
};
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
@@ -735,8 +735,12 @@ pub(crate) use core::io_primitives::disk_call_counters;
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::multipart::NewMultipartUploadCommitObservation;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause};
|
||||
#[cfg(test)]
|
||||
pub(crate) use ops::object::DeleteObjectCommitBarrier;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier;
|
||||
pub(crate) use ops::object::body_cache_plaintext_len;
|
||||
@@ -3025,6 +3029,16 @@ pub struct SetDisks {
|
||||
storage_class_config_override: Arc<std::sync::RwLock<Option<Arc<storageclass::Config>>>>,
|
||||
}
|
||||
|
||||
// DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones
|
||||
// each endpoint's canonical Arc, so an exact Arc set identifies the lock domain.
|
||||
pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
|
||||
left.iter()
|
||||
.all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client)))
|
||||
&& right
|
||||
.iter()
|
||||
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
|
||||
}
|
||||
|
||||
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
@@ -3600,6 +3614,15 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
/// Whether both sets' namespace-lock implementations cover the same object key.
|
||||
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
|
||||
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
|
||||
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
|
||||
(true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The lock manager this set actually uses (test-only; Phase 5 Slice 3).
|
||||
#[cfg(test)]
|
||||
pub(crate) fn local_lock_manager_for_test(&self) -> &Arc<rustfs_lock::GlobalLockManager> {
|
||||
@@ -4584,11 +4607,11 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool {
|
||||
}
|
||||
|
||||
fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool {
|
||||
opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement)
|
||||
opts.delete_marker || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.data_movement)
|
||||
}
|
||||
|
||||
fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) {
|
||||
let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none());
|
||||
let mut mark_delete = goi.version_id.is_some() || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none());
|
||||
let mut delete_marker = opts.versioned;
|
||||
|
||||
if opts.version_id.is_some() {
|
||||
|
||||
@@ -32,6 +32,8 @@ use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
use futures::{StreamExt, stream};
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::AtomicBool;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
@@ -65,6 +67,7 @@ impl StaleMultipartCleanupGuard {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MultipartCommitPause {
|
||||
NewUploadBeforeLockLost,
|
||||
PutPartBeforeLockAcquire,
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterRename,
|
||||
@@ -156,6 +159,72 @@ impl Drop for MultipartCommitBarrier {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct NewMultipartUploadCommitObservationState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
committed: AtomicBool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct NewMultipartUploadCommitObservation {
|
||||
state: Arc<NewMultipartUploadCommitObservationState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<NewMultipartUploadCommitObservationState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl NewMultipartUploadCommitObservation {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(NewMultipartUploadCommitObservationState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
committed: AtomicBool::new(false),
|
||||
});
|
||||
let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("new multipart upload commit observation mutex should not poison");
|
||||
assert!(slot.is_none(), "new multipart upload commit observation must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) fn committed(&self) -> bool {
|
||||
self.state.committed.load(Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for NewMultipartUploadCommitObservation {
|
||||
fn drop(&mut self) {
|
||||
let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("new multipart upload commit observation mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn observe_new_multipart_upload_commit(bucket: &str, object: &str) {
|
||||
let state = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("new multipart upload commit observation mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|state| state.bucket == bucket && state.object == object)
|
||||
.cloned();
|
||||
if let Some(state) = state {
|
||||
state.committed.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) {
|
||||
let barrier = {
|
||||
@@ -1615,6 +1684,30 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|
||||
let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement);
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::NewUploadBeforeLockLost).await;
|
||||
if _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "new_multipart_upload_commit",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "new_multipart_upload_outer_lock",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
Self::write_unique_file_info(
|
||||
&shuffle_disks,
|
||||
@@ -1626,6 +1719,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
)
|
||||
.await
|
||||
.map_err(|e| to_object_err(e.into(), vec![bucket, object]))?;
|
||||
#[cfg(test)]
|
||||
observe_new_multipart_upload_commit(bucket, object);
|
||||
|
||||
// evalDisks
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ use crate::bucket::lifecycle::{
|
||||
transition_transaction::{
|
||||
TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction,
|
||||
TransitionTransactionInit, TransitionTransactionState, delete_transition_transaction_record,
|
||||
save_transition_transaction_record,
|
||||
load_transition_transaction_record, save_transition_transaction_record,
|
||||
},
|
||||
};
|
||||
use crate::bucket::quota::reservation;
|
||||
@@ -45,6 +45,7 @@ use crate::bucket::replication::{
|
||||
DeleteReplicationConfigSnapshot, ReplicationLifecycleBridge, ReplicationStatusType, VersionPurgeStatusType,
|
||||
replication_state_to_filemeta, replication_status_from_filemeta, version_purge_status_to_filemeta,
|
||||
};
|
||||
use crate::data_usage::quota_object_size;
|
||||
use crate::diagnostics::get::GetObjectFailureReason;
|
||||
use crate::disk::{DataDirDeleteStatus, OldCurrentSize};
|
||||
use crate::error::is_err_invalid_upload_id;
|
||||
@@ -1293,7 +1294,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
(prepared.snapshot, prepared.object_info)
|
||||
} else {
|
||||
match self
|
||||
.get_object_fileinfo(
|
||||
.get_object_fileinfo_for_get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
@@ -2122,14 +2123,27 @@ impl SetDisks {
|
||||
let erasure = Arc::new(erasure_from_file_info(&fi, false)?);
|
||||
|
||||
let put_object_size = known_put_object_storage_size(data.size());
|
||||
let shard_file_size_raw = erasure.shard_file_size(put_object_size);
|
||||
let is_inline_buffer =
|
||||
storage_class_config.should_inline(erasure.shard_file_size(put_object_size), erasure.data_shards, opts.versioned);
|
||||
storage_class_config.should_inline(shard_file_size_raw, erasure.data_shards, opts.versioned);
|
||||
|
||||
let collect_stage_timing = rustfs_io_metrics::put_stage_metrics_enabled() || issue3031_diag_enabled();
|
||||
let shard_file_size = erasure.shard_file_size(put_object_size);
|
||||
let shard_file_size = shard_file_size_raw;
|
||||
let shard_size = erasure.shard_size();
|
||||
let write_path = classify_put_write_path(is_inline_buffer, put_object_size, fi.erasure.block_size);
|
||||
let direct_inline_commit = matches!(write_path, SmallWritePath::Inline);
|
||||
{
|
||||
use std::io::Write;
|
||||
let msg = format!(
|
||||
"INLINE_DEBUG: bucket={} obj={} size={} shard_fs={} ds={} bs={} inline={} direct={} path={} iblock={} ver={}\n",
|
||||
bucket, object, put_object_size, shard_file_size_raw, erasure.data_shards, fi.erasure.block_size,
|
||||
is_inline_buffer, direct_inline_commit, write_path.metric_label(), storage_class_config.inline_block(), opts.versioned
|
||||
);
|
||||
if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open("/tmp/rustfs_inline_debug.log") {
|
||||
let _ = f.write_all(msg.as_bytes());
|
||||
}
|
||||
let _ = std::io::stderr().write_all(msg.as_bytes());
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_path(write_path.metric_label());
|
||||
let writer_setup_stage_start = collect_stage_timing.then(Instant::now);
|
||||
let (mut writers, errors) = if direct_inline_commit {
|
||||
@@ -2483,6 +2497,7 @@ impl SetDisks {
|
||||
})
|
||||
.await?,
|
||||
);
|
||||
notify_put_object_commit_namespace_acquired(bucket, object);
|
||||
}
|
||||
#[cfg(not(any(test, feature = "test-util")))]
|
||||
{
|
||||
@@ -4246,7 +4261,12 @@ fn record_transition_uploaded_save_attempt(transaction: &TransitionTransaction,
|
||||
|
||||
async fn delete_transition_transaction_if_available(api: Option<&Arc<ECStore>>, transaction_id: Uuid) -> Result<()> {
|
||||
if let Some(api) = api {
|
||||
return delete_transition_transaction_record(api.clone(), transaction_id).await;
|
||||
let transaction = match load_transition_transaction_record(api.clone(), transaction_id).await {
|
||||
Ok(transaction) => transaction,
|
||||
Err(Error::ConfigNotFound) => return Ok(()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
return delete_transition_transaction_record(api.clone(), &transaction).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -4630,6 +4650,7 @@ struct PutObjectCommitBarrierState {
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
namespace_pending: tokio::sync::Notify,
|
||||
namespace_acquired: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
@@ -4651,6 +4672,7 @@ impl PutObjectCommitBarrier {
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
namespace_pending: tokio::sync::Notify::new(),
|
||||
namespace_acquired: std::sync::atomic::AtomicBool::new(false),
|
||||
});
|
||||
let mut slot = PUT_OBJECT_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
@@ -4685,6 +4707,10 @@ impl PutObjectCommitBarrier {
|
||||
.await
|
||||
.expect("put object should wait for the namespace lock after leaving the commit barrier");
|
||||
}
|
||||
|
||||
pub fn namespace_acquired(&self) -> bool {
|
||||
self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
@@ -4741,6 +4767,22 @@ fn notify_put_object_commit_namespace_pending(bucket: &str, object: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
|
||||
let barrier = PUT_OBJECT_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(Vec::new()))
|
||||
.lock()
|
||||
.expect("put object commit barrier mutex should not poison")
|
||||
.iter()
|
||||
.find(|barrier| {
|
||||
barrier.bucket == bucket && barrier.object == object && barrier.pause == PutObjectCommitPause::BeforeNamespace
|
||||
})
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DeleteObjectCommitBarrierState {
|
||||
bucket: String,
|
||||
@@ -4750,7 +4792,7 @@ struct DeleteObjectCommitBarrierState {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DeleteObjectCommitBarrier {
|
||||
pub(crate) struct DeleteObjectCommitBarrier {
|
||||
state: Arc<DeleteObjectCommitBarrierState>,
|
||||
}
|
||||
|
||||
@@ -4760,7 +4802,7 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
|
||||
|
||||
#[cfg(test)]
|
||||
impl DeleteObjectCommitBarrier {
|
||||
fn install(bucket: &str, object: &str) -> Self {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(DeleteObjectCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
@@ -4776,13 +4818,13 @@ impl DeleteObjectCommitBarrier {
|
||||
Self { state }
|
||||
}
|
||||
|
||||
async fn wait_until_paused(&self) {
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("delete object should reach the deterministic commit barrier");
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
pub(crate) fn release(&self) {
|
||||
self.state.release.notify_one();
|
||||
}
|
||||
}
|
||||
@@ -5655,7 +5697,18 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
|
||||
let (deleted, errors, _) = self.delete_objects_with_accounting(bucket, objects, opts).await;
|
||||
(deleted, errors)
|
||||
}
|
||||
|
||||
async fn delete_objects_with_accounting(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: Vec<ObjectToDelete>,
|
||||
opts: ObjectOptions,
|
||||
) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
|
||||
let mut del_objects = vec![DeletedObject::default(); objects.len()];
|
||||
let mut accounting = vec![None; objects.len()];
|
||||
let delete_config_snapshot = opts
|
||||
.delete_replication_config_snapshot
|
||||
.clone()
|
||||
@@ -5745,7 +5798,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
*item = Some(Error::other(message.clone()));
|
||||
}
|
||||
}
|
||||
return (del_objects, del_errs);
|
||||
return (del_objects, del_errs, accounting);
|
||||
}
|
||||
},
|
||||
}
|
||||
@@ -5792,6 +5845,22 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let source_missing = gerr
|
||||
.as_ref()
|
||||
.is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err));
|
||||
// Resolve accounting from the generation selected under this
|
||||
// object's write lock. A request-layer pre-stat is only an
|
||||
// optimization and cannot identify a concurrent overwrite.
|
||||
let (accounting_size, accounting_version_id, removed_current_object) = if source_missing
|
||||
|| dobj.synthetic_version_id
|
||||
|| set_disk_delete_creates_delete_marker(&check_opts)
|
||||
|| goi.delete_marker
|
||||
{
|
||||
(None, None, false)
|
||||
} else {
|
||||
(
|
||||
quota_object_size(&goi).ok(),
|
||||
goi.version_id.filter(|version_id| !version_id.is_nil()),
|
||||
(dobj.version_id.is_none() || is_explicit_null_version(dobj.version_id)) && !dobj.synthetic_version_id,
|
||||
)
|
||||
};
|
||||
// Normalize both sides before comparing. `goi.version_id` is the
|
||||
// client-facing identity, where `from_file_info` synthesizes
|
||||
// `Some(Uuid::nil())` for a null version on a versioned or
|
||||
@@ -5877,6 +5946,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
if dobj.version_id.is_none() && (version_suspended || versioned) {
|
||||
vr.mod_time = Some(OffsetDateTime::now_utc());
|
||||
vr.deleted = true;
|
||||
vr.mark_deleted = true;
|
||||
if versioned {
|
||||
vr.version_id = Some(Uuid::new_v4());
|
||||
}
|
||||
@@ -5920,7 +5990,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
},
|
||||
replication_state: vr.replication_state_internal.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
};
|
||||
accounting[i] = Some(DeleteAccounting {
|
||||
size: accounting_size,
|
||||
version_id: accounting_version_id,
|
||||
removed_current_object,
|
||||
});
|
||||
}
|
||||
|
||||
// Only add to vers_map if we hold the lock
|
||||
@@ -5966,7 +6041,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
});
|
||||
}
|
||||
}
|
||||
return (del_objects, del_errs);
|
||||
return (del_objects, del_errs, accounting);
|
||||
}
|
||||
|
||||
let mut persisted_journal_entries = Vec::with_capacity(journal_entries.len());
|
||||
@@ -6204,7 +6279,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
(del_objects, del_errs)
|
||||
// An accounting identity is actionable only when the delete result is
|
||||
// successful. Never let a failed commit (including a partial quorum
|
||||
// failure) reach the request-layer fast delta path.
|
||||
for (index, err) in del_errs.iter().enumerate() {
|
||||
if err.is_some() {
|
||||
accounting[index] = None;
|
||||
}
|
||||
}
|
||||
|
||||
(del_objects, del_errs, accounting)
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
@@ -6533,6 +6617,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
let mut obj_info = ObjectInfo::from_file_info(&dfi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
obj_info.size = goi.size;
|
||||
// Keep the committed source metadata on the internal delete result so
|
||||
// the request layer can derive canonical accounting for this exact
|
||||
// generation. Delete responses do not expose these fields.
|
||||
obj_info.actual_size = goi.actual_size;
|
||||
obj_info.user_defined = Arc::clone(&goi.user_defined);
|
||||
obj_info.parts = Arc::clone(&goi.parts);
|
||||
obj_info.user_tags = Arc::clone(&goi.user_tags);
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
Ok(obj_info)
|
||||
@@ -7824,6 +7914,113 @@ mod replication_quota_safety_tests {
|
||||
assert_eq!(stored.get_actual_size().expect("stored logical size should parse"), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_returns_canonical_compressed_accounting_size() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "compressed-delete-accounting";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
|
||||
let mut reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
|
||||
.expect("compressed fixture reader should be valid"),
|
||||
);
|
||||
set_disks
|
||||
.put_object(
|
||||
bucket,
|
||||
"object",
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
user_defined,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("compressed object should be written");
|
||||
|
||||
let (deleted, errors, accounting) = set_disks
|
||||
.delete_objects_with_accounting(
|
||||
bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: "object".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
|
||||
ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(errors[0].is_none(), "compressed delete should succeed: {:?}", errors[0]);
|
||||
assert!(deleted[0].found, "the committed object must be reported as found");
|
||||
assert_eq!(accounting[0].as_ref().and_then(|value| value.size), Some(1000));
|
||||
assert!(accounting[0].as_ref().is_some_and(|value| value.version_id.is_none()));
|
||||
assert!(accounting[0].as_ref().is_some_and(|value| value.removed_current_object));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn suspended_delete_marker_does_not_return_body_accounting() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "suspended-delete-accounting";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
insert_str(&mut user_defined, SUFFIX_ACTUAL_SIZE, "1000".to_string());
|
||||
let mut reader = PutObjReader::new(
|
||||
HashReader::from_stream(Cursor::new(vec![0x5a; 400]), 400, 1000, None, None, false)
|
||||
.expect("compressed fixture reader should be valid"),
|
||||
);
|
||||
let suspended_opts = ObjectOptions {
|
||||
version_suspended: true,
|
||||
delete_replication_config_snapshot: Some(Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
|
||||
s3s::dto::VersioningConfiguration {
|
||||
status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)),
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
))),
|
||||
user_defined,
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
};
|
||||
set_disks
|
||||
.put_object(bucket, "object", &mut reader, &suspended_opts)
|
||||
.await
|
||||
.expect("compressed object should be written");
|
||||
|
||||
let (deleted, errors, accounting) = set_disks
|
||||
.delete_objects_with_accounting(
|
||||
bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: "object".to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
suspended_opts,
|
||||
)
|
||||
.await;
|
||||
assert!(errors[0].is_none(), "suspended delete should create a marker: {:?}", errors[0]);
|
||||
assert!(deleted[0].delete_marker);
|
||||
assert!(accounting[0].is_none(), "a delete marker must not carry body accounting");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn direct_put_cannot_persist_a_tiny_logical_size() {
|
||||
let (_temp_dirs, disks, set_disks) = hermetic_set_disks(4).await;
|
||||
@@ -11636,6 +11833,7 @@ mod transition_upload_integrity_tests {
|
||||
crate::data_movement::SourceCleanupBucketFence {
|
||||
expected_incarnation_id: None,
|
||||
lifecycle_guard: Some(&bucket_guard),
|
||||
..Default::default()
|
||||
},
|
||||
"test_data_movement",
|
||||
)
|
||||
|
||||
@@ -259,10 +259,33 @@ impl SetDisks {
|
||||
read_data: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop)
|
||||
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
pub(super) async fn get_object_fileinfo_for_get_object_reader(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
caller_allows_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket)
|
||||
&& crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
|
||||
self.get_object_fileinfo_gated_inner(
|
||||
bucket,
|
||||
object,
|
||||
opts,
|
||||
read_data,
|
||||
caller_allows_early_stop,
|
||||
allow_read_version_coalescing,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full
|
||||
/// quorum fanout. Read-before-write callers (object tagging) must use this:
|
||||
/// the returned online-disk set is the write target, and the early-stop
|
||||
@@ -275,6 +298,20 @@ impl SetDisks {
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn get_object_fileinfo_gated_inner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
read_data: bool,
|
||||
allow_early_stop: bool,
|
||||
allow_read_version_coalescing: bool,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
let vid = opts.version_id.clone().unwrap_or_default();
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
@@ -337,19 +374,34 @@ impl SetDisks {
|
||||
// read_all_fileinfo_observed (see read_all_fileinfo_early_stop in
|
||||
// core/io_primitives.rs); unsafe requests and callers that opt out
|
||||
// (allow_early_stop=false) fall back to full-wait.
|
||||
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
false,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?;
|
||||
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing {
|
||||
Self::read_all_fileinfo_observed_for_get_object(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
Self::read_all_fileinfo_observed(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
false,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?
|
||||
};
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
|
||||
@@ -62,8 +62,8 @@ pub(crate) mod object {
|
||||
use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions, ObjectOperations,
|
||||
ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
|
||||
DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions,
|
||||
ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
|
||||
};
|
||||
|
||||
pub(crate) trait EcstoreObjectIO:
|
||||
|
||||
@@ -297,10 +297,16 @@ impl ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
|
||||
use crate::disk::{DiskOption, format::FormatV3, new_disk};
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
use crate::disk::{DeleteOptions, DiskOption, format::FormatV3, new_disk};
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
|
||||
use crate::store::init_format::{load_format_erasure, save_format_file};
|
||||
use crate::store::init_local_disks_with_instance_ctx;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
|
||||
let format = FormatV3::new(1, 1);
|
||||
@@ -347,6 +353,51 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
|
||||
let mut pool_endpoints = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("multi-pool heal test disk should be created");
|
||||
let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
|
||||
.expect("test endpoint should parse");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
pool_endpoints.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("heal-owner-pool-{pool_index}"),
|
||||
platform: "test".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(pool_endpoints);
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
.await
|
||||
.expect("multi-pool local disks should initialize");
|
||||
let shutdown = CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx(
|
||||
"127.0.0.1:0".parse().expect("test address should parse"),
|
||||
endpoint_pools,
|
||||
shutdown.clone(),
|
||||
instance_ctx,
|
||||
)
|
||||
.await
|
||||
.expect("multi-pool test store should initialize");
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
(temp_dir, store, shutdown)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_object_pool_scope_selects_only_requested_pool() {
|
||||
let store = minimal_heal_store().await;
|
||||
@@ -506,6 +557,229 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn unscoped_heal_object_suspended_owner_semantics() {
|
||||
let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
|
||||
let active_object = "active-owner";
|
||||
let suspended_only_object = "suspended-only";
|
||||
let duplicate_object = "duplicate-owner";
|
||||
let marker_object = "marker-owner";
|
||||
let quorum_object = "quorum-owner";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created in all pools");
|
||||
|
||||
let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("active owner object should be written");
|
||||
let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
|
||||
missing_active_disk
|
||||
.delete(
|
||||
&bucket,
|
||||
active_object,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("active owner shard should be removed for repair");
|
||||
assert!(
|
||||
missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
|
||||
"the active owner fixture must start with one missing metadata copy"
|
||||
);
|
||||
|
||||
let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
|
||||
store.pools[1]
|
||||
.put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("suspended owner object should be written");
|
||||
for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
|
||||
let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
|
||||
store.pools[pool_index]
|
||||
.put_object(
|
||||
&bucket,
|
||||
duplicate_object,
|
||||
&mut duplicate_reader,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("duplicate owner object should be written");
|
||||
}
|
||||
let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
|
||||
.clone()
|
||||
.expect("duplicate active owner disk should be online");
|
||||
duplicate_missing_disk
|
||||
.delete(
|
||||
&bucket,
|
||||
duplicate_object,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("duplicate active owner shard should be removed for repair");
|
||||
let history_version = Uuid::new_v4();
|
||||
let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
marker_object,
|
||||
&mut history_reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(history_version.to_string()),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("versioned marker history should be written");
|
||||
store.pools[0]
|
||||
.delete_object(
|
||||
&bucket,
|
||||
marker_object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("delete marker should be written");
|
||||
let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
|
||||
store.pools[0]
|
||||
.put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("quorum boundary object should be written");
|
||||
{
|
||||
let mut pool_meta = store.pool_meta.write().await;
|
||||
let mut next = PoolMeta::new(&store.pools, &pool_meta);
|
||||
next.pools[1].decommission = Some(PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
});
|
||||
*pool_meta = next;
|
||||
}
|
||||
|
||||
let (_, duplicate_owner) = store
|
||||
.get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("duplicate owner should resolve");
|
||||
assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
|
||||
let (_, active_duplicate_owner) = store
|
||||
.get_latest_object_info_with_idx(
|
||||
&bucket,
|
||||
duplicate_object,
|
||||
&ObjectOptions {
|
||||
skip_decommissioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("active duplicate owner should resolve");
|
||||
assert_eq!(
|
||||
active_duplicate_owner, 0,
|
||||
"suspended duplicate must be excluded from active owner selection"
|
||||
);
|
||||
let (duplicate_result, duplicate_err) = store
|
||||
.handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("duplicate owner heal should complete through the production path");
|
||||
assert_eq!(duplicate_result.object, duplicate_object);
|
||||
assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
|
||||
assert!(
|
||||
duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
|
||||
"production heal must repair the active duplicate owner rather than the suspended owner"
|
||||
);
|
||||
let (marker_info, marker_owner) = store
|
||||
.get_latest_object_info_with_idx(
|
||||
&bucket,
|
||||
marker_object,
|
||||
&ObjectOptions {
|
||||
skip_decommissioned: true,
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("latest delete marker should resolve");
|
||||
assert_eq!(marker_owner, 0);
|
||||
assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
|
||||
|
||||
let (active_result, active_err) = store
|
||||
.handle_heal_object(&bucket, active_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("unscoped active-owner heal should complete");
|
||||
assert_eq!(active_result.object, active_object);
|
||||
assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
|
||||
assert!(
|
||||
missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
|
||||
"active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
|
||||
);
|
||||
assert!(
|
||||
store.pools[1]
|
||||
.get_object_info(&bucket, active_object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"the suspended pool must not be written for an active-owner object"
|
||||
);
|
||||
|
||||
let (suspended_result, suspended_err) = store
|
||||
.handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("unscoped suspended-only heal should return a terminal result");
|
||||
assert!(suspended_result.object.is_empty());
|
||||
assert!(matches!(suspended_err, Some(Error::FileNotFound)));
|
||||
assert!(
|
||||
store.pools[1]
|
||||
.get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
|
||||
.await
|
||||
.is_ok(),
|
||||
"suspended-only data must remain untouched when unscoped heal reports absent"
|
||||
);
|
||||
|
||||
let (_, explicit_err) = store
|
||||
.handle_heal_object(
|
||||
&bucket,
|
||||
suspended_only_object,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("explicit suspended-owner heal should return a mapped error");
|
||||
assert!(matches!(explicit_err, Some(Error::SlowDown)));
|
||||
|
||||
let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
|
||||
let surviving_quorum_disk = original_quorum_disks[3].clone();
|
||||
*store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
|
||||
let (_, quorum_err) = store
|
||||
.handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
|
||||
.await
|
||||
.expect("quorum boundary heal should return a mapped result");
|
||||
*store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
|
||||
assert!(
|
||||
matches!(quorum_err, Some(Error::ErasureReadQuorum)),
|
||||
"quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handle_heal_format_continues_after_a_pool_error() {
|
||||
let canonical_format = FormatV3::new(1, 3);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -33,7 +33,7 @@ use crate::bucket::utils::check_put_object_part_args;
|
||||
use crate::bucket::utils::{check_valid_bucket_name, check_valid_bucket_name_strict, is_meta_bucketname};
|
||||
use crate::cluster::rpc::{RemoteClient, S3PeerSys};
|
||||
use crate::config::storageclass;
|
||||
use crate::core::pools::PoolMeta;
|
||||
use crate::core::pools::{DecommissionCanceler, PoolMeta};
|
||||
use crate::disk::endpoint::{Endpoint, EndpointType};
|
||||
use crate::disk::{DiskAPI, DiskInfo, DiskInfoOptions};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -151,7 +151,7 @@ pub(crate) mod init_format;
|
||||
pub(crate) mod list_objects;
|
||||
mod multipart;
|
||||
mod object;
|
||||
pub(crate) use object::ObjectLockDiagGuard;
|
||||
pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence};
|
||||
pub use object::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
SnapshotConsistencyError,
|
||||
@@ -176,7 +176,7 @@ pub struct ECStore {
|
||||
// pub local_disks: Vec<DiskStore>,
|
||||
pub pool_meta: RwLock<PoolMeta>,
|
||||
pub rebalance_meta: RwLock<Option<RebalanceMeta>>,
|
||||
pub decommission_cancelers: RwLock<Vec<Option<CancellationToken>>>,
|
||||
pub decommission_cancelers: RwLock<Vec<Option<DecommissionCanceler>>>,
|
||||
/// Serializes rebalance/decommission start transitions.
|
||||
///
|
||||
/// Lock order: acquire `start_gate` before `pool_meta`, `rebalance_meta`,
|
||||
|
||||
@@ -400,7 +400,7 @@ impl ECStore {
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<MultipartUploadResult> {
|
||||
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts)
|
||||
self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts, None)
|
||||
.await
|
||||
.map(|(res, _, _)| res)
|
||||
}
|
||||
@@ -410,20 +410,22 @@ impl ECStore {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<(MultipartUploadResult, usize, Option<Uuid>)> {
|
||||
check_new_multipart_args(bucket, object)?;
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
if self.single_pool() {
|
||||
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
return self.pools[0]
|
||||
.new_multipart_upload(bucket, object, opts)
|
||||
.new_multipart_upload(bucket, object, &opts)
|
||||
.await
|
||||
.map(|res| (res, 0, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
if opts.data_movement && opts.version_id.is_some() {
|
||||
let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?;
|
||||
let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?;
|
||||
if idx == opts.src_pool_idx {
|
||||
return Err(StorageError::DataMovementOverwriteErr(
|
||||
bucket.to_owned(),
|
||||
@@ -431,7 +433,9 @@ impl ECStore {
|
||||
opts.version_id.clone().unwrap_or_default(),
|
||||
));
|
||||
}
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
|
||||
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
@@ -454,7 +458,9 @@ impl ECStore {
|
||||
.await?;
|
||||
|
||||
if !res.uploads.is_empty() {
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
|
||||
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
}
|
||||
@@ -467,7 +473,9 @@ impl ECStore {
|
||||
));
|
||||
}
|
||||
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?;
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
|
||||
Ok((res, idx, opts.expected_bucket_incarnation_id))
|
||||
}
|
||||
|
||||
@@ -704,13 +712,14 @@ impl ECStore {
|
||||
|
||||
pub(crate) async fn complete_multipart_upload_for_data_movement(
|
||||
self: Arc<Self>,
|
||||
target_pool_idx: usize,
|
||||
target: (usize, Option<&ObjectLockDiagGuard>),
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
uploaded_parts: Vec<CompletePart>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let (target_pool_idx, mutation_fence) = target;
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||
@@ -739,6 +748,8 @@ impl ECStore {
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
opts.object_lock_config_snapshot = Some(snapshot);
|
||||
}
|
||||
self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
#[cfg(test)]
|
||||
pause_data_movement_multipart_before_selected_completion(bucket).await;
|
||||
let pool = self
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -859,6 +859,7 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
|
||||
use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
@@ -1423,6 +1424,14 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
version_id: Some(version_id),
|
||||
etag,
|
||||
..object_info_with_mod_time(unix_ts, delete_marker)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
|
||||
let candidates = vec![
|
||||
@@ -1446,7 +1455,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() {
|
||||
fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_mod_time(10, false)),
|
||||
@@ -1466,6 +1475,382 @@ mod tests {
|
||||
assert_eq!(idx, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 2,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 7,
|
||||
err: None,
|
||||
},
|
||||
];
|
||||
|
||||
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect("equivalent replicas must resolve deterministically");
|
||||
|
||||
assert_eq!(idx, 7);
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
];
|
||||
|
||||
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect_err("divergent version ids must not silently resolve to the higher pool index");
|
||||
|
||||
assert_eq!(err, Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
];
|
||||
|
||||
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect_err("divergent etags must not silently resolve to the higher pool index");
|
||||
|
||||
assert_eq!(err, Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
];
|
||||
|
||||
let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect_err("a delete marker tied with a live version must not be masked by the pool index");
|
||||
|
||||
assert_eq!(err, Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
|
||||
let err = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(left),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(right),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect_err("equal-time identity divergence must fail closed");
|
||||
|
||||
assert_eq!(err, Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
|
||||
let mut data_dir = base.clone();
|
||||
data_dir.data_dir = Some(Uuid::from_u128(2));
|
||||
assert_equal_time_identity_conflict(base.clone(), data_dir);
|
||||
|
||||
let mut size = base.clone();
|
||||
size.size = 1;
|
||||
assert_equal_time_identity_conflict(base.clone(), size);
|
||||
|
||||
let mut actual_size = base.clone();
|
||||
actual_size.actual_size = 1;
|
||||
assert_equal_time_identity_conflict(base.clone(), actual_size);
|
||||
|
||||
let mut checksum = base.clone();
|
||||
checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
|
||||
assert_equal_time_identity_conflict(base.clone(), checksum);
|
||||
|
||||
let mut parts = base.clone();
|
||||
parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
|
||||
etag: "part-etag".to_string(),
|
||||
number: 1,
|
||||
size: 1,
|
||||
..Default::default()
|
||||
}]);
|
||||
assert_equal_time_identity_conflict(base.clone(), parts);
|
||||
|
||||
let mut transition = base;
|
||||
transition.transitioned_object.tier = "tier-a".to_string();
|
||||
assert_equal_time_identity_conflict(
|
||||
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
|
||||
transition,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
let mut rustfs_alias = base.clone();
|
||||
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"x-rustfs-internal-compression".to_string(),
|
||||
"zstd".to_string(),
|
||||
)]));
|
||||
let mut minio_alias = base.clone();
|
||||
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"X-MINIO-INTERNAL-COMPRESSION".to_string(),
|
||||
"zstd".to_string(),
|
||||
)]));
|
||||
|
||||
let (_, idx) = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(rustfs_alias),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(minio_alias),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect("same-value internal aliases should resolve");
|
||||
assert_eq!(idx, 1);
|
||||
|
||||
let mut dual_alias = base.clone();
|
||||
dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
|
||||
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
|
||||
("x-minio-internal-compression".to_string(), "zstd".to_string()),
|
||||
]));
|
||||
let mut single_alias = base;
|
||||
single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"x-rustfs-internal-compression".to_string(),
|
||||
"zstd".to_string(),
|
||||
)]));
|
||||
|
||||
let (_, idx) = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(dual_alias),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(single_alias),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect("dual-key and single-key internal metadata should resolve");
|
||||
assert_eq!(idx, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
let mut rustfs_alias = base.clone();
|
||||
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"x-rustfs-internal-compression".to_string(),
|
||||
"zstd".to_string(),
|
||||
)]));
|
||||
let mut minio_alias = base;
|
||||
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
"x-minio-internal-compression".to_string(),
|
||||
"snappy".to_string(),
|
||||
)]));
|
||||
|
||||
assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
|
||||
for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
let mut rustfs_alias = base.clone();
|
||||
rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
format!(
|
||||
"X-RUSTFS-INTERNAL-{}{suffix}",
|
||||
suffix_prefix.to_uppercase(),
|
||||
suffix = "arn:aws:s3:::Bucket"
|
||||
),
|
||||
"value".to_string(),
|
||||
)]));
|
||||
let mut minio_alias = base.clone();
|
||||
minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
|
||||
"value".to_string(),
|
||||
)]));
|
||||
|
||||
let (_, idx) = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(rustfs_alias.clone()),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(minio_alias),
|
||||
idx: 1,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect("dynamic internal aliases with the same target should resolve");
|
||||
assert_eq!(idx, 1);
|
||||
|
||||
let mut different_target_case = base;
|
||||
different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
|
||||
format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
|
||||
"value".to_string(),
|
||||
)]));
|
||||
|
||||
assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
let mut first = base.clone();
|
||||
first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
|
||||
("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
|
||||
("x-minio-internal-compression".to_string(), "snappy".to_string()),
|
||||
]));
|
||||
let mut second = base;
|
||||
second.user_defined = first.user_defined.clone();
|
||||
|
||||
assert_equal_time_identity_conflict(first, second);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
|
||||
let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
|
||||
let mut replication = base.clone();
|
||||
replication.replication_status_internal = Some("PENDING".to_string());
|
||||
replication.replication_status = ReplicationStatusType::Pending;
|
||||
assert_equal_time_identity_conflict(base.clone(), replication);
|
||||
|
||||
let mut purge = base.clone();
|
||||
purge.version_purge_status_internal = Some("PENDING".to_string());
|
||||
purge.version_purge_status = VersionPurgeStatusType::Pending;
|
||||
assert_equal_time_identity_conflict(base.clone(), purge);
|
||||
|
||||
let mut decision = base;
|
||||
decision.replication_decision = "replicate".to_string();
|
||||
assert_equal_time_identity_conflict(
|
||||
object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
|
||||
decision,
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
|
||||
let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
without_mod_time.mod_time = None;
|
||||
let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
|
||||
|
||||
assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
|
||||
let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
|
||||
let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
|
||||
older.data_dir = Some(Uuid::from_u128(2));
|
||||
|
||||
let (info, idx) = resolve_latest_object_info_candidates(
|
||||
vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(latest),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(older),
|
||||
idx: 9,
|
||||
err: None,
|
||||
},
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.expect("older identity divergence must not affect the latest candidate");
|
||||
|
||||
assert_eq!(idx, 0);
|
||||
assert_eq!(
|
||||
info.mod_time,
|
||||
Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
|
||||
let candidates = vec![
|
||||
LatestObjectInfoCandidate {
|
||||
info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
|
||||
idx: 0,
|
||||
err: None,
|
||||
},
|
||||
LatestObjectInfoCandidate {
|
||||
info: None,
|
||||
idx: 1,
|
||||
err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
|
||||
},
|
||||
];
|
||||
|
||||
let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
|
||||
.expect("not-found pools must not block resolution of found candidates");
|
||||
|
||||
assert_eq!(idx, 0);
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
|
||||
let err = resolve_latest_object_info_candidates(
|
||||
|
||||
@@ -12,10 +12,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use rustfs_utils::http::metadata_compat::{
|
||||
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX,
|
||||
strip_internal_prefix_preserving_case,
|
||||
};
|
||||
use rustfs_utils::path::decode_dir_object;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
@@ -73,7 +77,7 @@ pub(super) fn resolve_rebalance_delete_from_all_pools_result(
|
||||
object: &str,
|
||||
) -> Result<ObjectInfo> {
|
||||
result.map_err(|err| {
|
||||
if err == Error::PreconditionFailed {
|
||||
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
|
||||
err
|
||||
} else {
|
||||
Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}"))
|
||||
@@ -86,7 +90,7 @@ fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
|
||||
}
|
||||
|
||||
fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
|
||||
if err == Error::PreconditionFailed {
|
||||
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
|
||||
err
|
||||
} else {
|
||||
Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
|
||||
@@ -137,37 +141,158 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p
|
||||
))
|
||||
}
|
||||
|
||||
fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option<OffsetDateTime> {
|
||||
candidate
|
||||
.info
|
||||
.as_ref()
|
||||
.map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
||||
}
|
||||
|
||||
fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
|
||||
left.transition_version_state == right.transition_version_state
|
||||
&& left.transitioned_object.name == right.transitioned_object.name
|
||||
&& left.transitioned_object.version_id == right.transitioned_object.version_id
|
||||
&& left.transitioned_object.tier == right.transitioned_object.tier
|
||||
&& left.transitioned_object.free_version == right.transitioned_object.free_version
|
||||
&& left.transitioned_object.status == right.transitioned_object.status
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq)]
|
||||
struct LatestUserDefinedIdentity {
|
||||
internal: HashMap<String, String>,
|
||||
other: HashMap<String, String>,
|
||||
}
|
||||
|
||||
fn normalize_internal_identity_suffix(key: &str) -> Option<String> {
|
||||
let suffix = strip_internal_prefix_preserving_case(key)?;
|
||||
|
||||
for dynamic_prefix in [
|
||||
SUFFIX_REPLICATION_RESET_ARN_PREFIX,
|
||||
SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
|
||||
] {
|
||||
let prefix_len = dynamic_prefix.len();
|
||||
if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..))
|
||||
&& prefix.eq_ignore_ascii_case(dynamic_prefix)
|
||||
{
|
||||
return Some(format!("{dynamic_prefix}{remainder}"));
|
||||
}
|
||||
}
|
||||
|
||||
Some(suffix.to_lowercase())
|
||||
}
|
||||
|
||||
fn normalize_user_defined_identity(user_defined: &HashMap<String, String>) -> Option<LatestUserDefinedIdentity> {
|
||||
let mut identity = LatestUserDefinedIdentity {
|
||||
internal: HashMap::with_capacity(user_defined.len()),
|
||||
other: HashMap::with_capacity(user_defined.len()),
|
||||
};
|
||||
|
||||
for (key, value) in user_defined {
|
||||
if let Some(suffix) = normalize_internal_identity_suffix(key) {
|
||||
if identity
|
||||
.internal
|
||||
.insert(suffix, value.clone())
|
||||
.is_some_and(|previous| previous != *value)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
} else {
|
||||
identity.other.insert(key.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Some(identity)
|
||||
}
|
||||
|
||||
fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
|
||||
match (
|
||||
normalize_user_defined_identity(&left.user_defined),
|
||||
normalize_user_defined_identity(&right.user_defined),
|
||||
) {
|
||||
(Some(left), Some(right)) => left == right,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Pool-specific erasure geometry is intentionally excluded: `get_object_info`
|
||||
/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can
|
||||
/// differ for the same object version while the selected winner still carries
|
||||
/// the chosen pool's layout. `put_object_reader` is also intentionally
|
||||
/// excluded because it is a transient request handle that `ObjectInfo::clone`
|
||||
/// drops. Every other ObjectInfo field is part of the production-visible
|
||||
/// identity and must agree before the pool index can provide a deterministic
|
||||
/// tie-break.
|
||||
fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
|
||||
left.bucket == right.bucket
|
||||
&& left.name == right.name
|
||||
&& left.storage_class == right.storage_class
|
||||
&& left.mod_time == right.mod_time
|
||||
&& left.size == right.size
|
||||
&& left.actual_size == right.actual_size
|
||||
&& left.is_dir == right.is_dir
|
||||
&& same_user_defined_identity(left, right)
|
||||
&& left.user_tags == right.user_tags
|
||||
&& left.version_id == right.version_id
|
||||
&& left.data_dir == right.data_dir
|
||||
&& left.delete_marker == right.delete_marker
|
||||
&& same_transition_identity(left, right)
|
||||
&& left.restore_ongoing == right.restore_ongoing
|
||||
&& left.restore_expires == right.restore_expires
|
||||
&& left.parts == right.parts
|
||||
&& left.is_latest == right.is_latest
|
||||
&& left.content_type == right.content_type
|
||||
&& left.content_encoding == right.content_encoding
|
||||
&& left.expires == right.expires
|
||||
&& left.num_versions == right.num_versions
|
||||
&& left.successor_mod_time == right.successor_mod_time
|
||||
&& left.etag == right.etag
|
||||
&& left.inlined == right.inlined
|
||||
&& left.metadata_only == right.metadata_only
|
||||
&& left.version_only == right.version_only
|
||||
&& left.replication_status_internal == right.replication_status_internal
|
||||
&& left.replication_status == right.replication_status
|
||||
&& left.version_purge_status_internal == right.version_purge_status_internal
|
||||
&& left.version_purge_status == right.version_purge_status
|
||||
&& left.replication_decision == right.replication_decision
|
||||
&& left.checksum == right.checksum
|
||||
}
|
||||
|
||||
pub(super) fn resolve_latest_object_info_candidates(
|
||||
mut candidates: Vec<LatestObjectInfoCandidate>,
|
||||
candidates: Vec<LatestObjectInfoCandidate>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
candidates.sort_by(|a, b| {
|
||||
let a_mod = if let Some(info) = &a.info {
|
||||
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
} else {
|
||||
OffsetDateTime::UNIX_EPOCH
|
||||
let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max();
|
||||
|
||||
if let Some(latest_mod_time) = latest_mod_time {
|
||||
let mut latest_candidates = candidates
|
||||
.into_iter()
|
||||
.filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx));
|
||||
|
||||
let Some(winner) = latest_candidates.first() else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
};
|
||||
let Some(winner_info) = winner.info.as_ref() else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
};
|
||||
|
||||
let b_mod = if let Some(info) = &b.info {
|
||||
info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)
|
||||
} else {
|
||||
OffsetDateTime::UNIX_EPOCH
|
||||
};
|
||||
|
||||
if a_mod == b_mod {
|
||||
return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less };
|
||||
if latest_candidates.iter().skip(1).any(|candidate| {
|
||||
candidate
|
||||
.info
|
||||
.as_ref()
|
||||
.is_none_or(|info| !same_latest_object_info_identity(winner_info, info))
|
||||
}) {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
b_mod.cmp(&a_mod)
|
||||
});
|
||||
return Ok((winner_info.clone(), winner.idx));
|
||||
}
|
||||
|
||||
for candidate in candidates {
|
||||
if let Some(info) = candidate.info {
|
||||
return Ok((info, candidate.idx));
|
||||
}
|
||||
|
||||
if let Some(err) = candidate.err
|
||||
&& !is_err_object_not_found(&err)
|
||||
&& !is_err_version_not_found(&err)
|
||||
@@ -191,6 +316,18 @@ mod tests {
|
||||
assert_eq!(err, Error::PreconditionFailed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_result_preserves_prefix_access_denied() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_result(
|
||||
Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("prefix access denial should remain structured");
|
||||
|
||||
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_pool_result_preserves_precondition_failed() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_results(
|
||||
@@ -205,4 +342,19 @@ mod tests {
|
||||
|
||||
assert_eq!(err, Error::PreconditionFailed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_pool_result_preserves_prefix_access_denied() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_results(
|
||||
vec![RebalanceDeletePoolResult {
|
||||
pool_idx: 0,
|
||||
result: Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
|
||||
}],
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("prefix access denial should remain structured");
|
||||
|
||||
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1640,6 +1640,60 @@ mod tests {
|
||||
assert_eq!(payload["items"].as_array().expect("items should be an array").len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_process_query_request_reports_displaced_terminal_detail() {
|
||||
let heal_manager = Arc::new(HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
));
|
||||
let mut displaced = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "displaced-channel".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
);
|
||||
displaced.id = "displaced-channel-task".to_string();
|
||||
let displaced_id = displaced.id.clone();
|
||||
heal_manager
|
||||
.submit_heal_request(displaced)
|
||||
.await
|
||||
.expect("initial channel task should queue");
|
||||
heal_manager
|
||||
.submit_heal_request(HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "successor-channel".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
))
|
||||
.await
|
||||
.expect("successor channel task should displace the initial task");
|
||||
|
||||
let processor = HealChannelProcessor::new(heal_manager);
|
||||
let (tx, rx) = oneshot::channel();
|
||||
processor
|
||||
.process_query_request("displaced-channel".to_string(), displaced_id, None, tx)
|
||||
.await
|
||||
.expect("displaced query should process");
|
||||
let response = rx
|
||||
.await
|
||||
.expect("query response should be returned")
|
||||
.expect("displaced query should remain successful");
|
||||
let payload: serde_json::Value = serde_json::from_slice(response.data.as_deref().expect("status payload should exist"))
|
||||
.expect("status payload should be json");
|
||||
assert_eq!(payload["summary"], "stopped");
|
||||
assert!(
|
||||
response
|
||||
.error
|
||||
.as_deref()
|
||||
.is_some_and(|detail| detail.contains("reason=displaced"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_process_query_request_reports_running_for_queued_task() {
|
||||
let heal_manager = create_test_heal_manager();
|
||||
|
||||
+105
-10
@@ -40,6 +40,7 @@ use tracing::{debug, error, info, warn};
|
||||
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
|
||||
|
||||
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
|
||||
const DISPLACED_HEAL_REASON: &str = "reason=displaced; retry_hint=submit_again";
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_DISK_SCANNER: &str = "disk_scanner";
|
||||
const LOG_SUBSYSTEM_MANAGER: &str = "manager";
|
||||
@@ -120,26 +121,30 @@ struct MrfRepairNoticeTarget {
|
||||
version_id: Option<[u8; 16]>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone)]
|
||||
struct HealAdmissionDecision {
|
||||
result: HealAdmissionResult,
|
||||
displaced_task_id: Option<String>,
|
||||
displaced_request: Option<HealRequest>,
|
||||
}
|
||||
|
||||
impl HealAdmissionDecision {
|
||||
const fn new(result: HealAdmissionResult) -> Self {
|
||||
Self {
|
||||
result,
|
||||
displaced_task_id: None,
|
||||
displaced_request: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn accepted_with_displacement(displaced_task_id: String) -> Self {
|
||||
fn accepted_with_displacement(displaced_request: HealRequest) -> Self {
|
||||
Self {
|
||||
result: HealAdmissionResult::Accepted,
|
||||
displaced_task_id: Some(displaced_task_id),
|
||||
displaced_request: Some(displaced_request),
|
||||
}
|
||||
}
|
||||
|
||||
fn displaced_task_id(&self) -> Option<&str> {
|
||||
self.displaced_request.as_ref().map(|request| request.id.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_mrf_repair_notice_targets(
|
||||
@@ -151,6 +156,55 @@ fn lock_mrf_repair_notice_targets(
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_displaced_terminals(
|
||||
registry: &StdMutex<HashMap<String, Arc<CompletedHealStatus>>>,
|
||||
) -> StdMutexGuard<'_, HashMap<String, Arc<CompletedHealStatus>>> {
|
||||
match registry.lock() {
|
||||
Ok(guard) => guard,
|
||||
Err(poisoned) => poisoned.into_inner(),
|
||||
}
|
||||
}
|
||||
|
||||
fn record_displaced_terminal(
|
||||
registry: &StdMutex<HashMap<String, Arc<CompletedHealStatus>>>,
|
||||
request: &HealRequest,
|
||||
) -> Arc<CompletedHealStatus> {
|
||||
let terminal = Arc::new(CompletedHealStatus {
|
||||
heal_type: request.heal_type.clone(),
|
||||
status: HealTaskStatus::Failed {
|
||||
error: format!("heal task displaced by a higher-priority request ({DISPLACED_HEAL_REASON})"),
|
||||
},
|
||||
result_items_truncated: false,
|
||||
completed_at: SystemTime::now(),
|
||||
seqed_items: Vec::new(),
|
||||
next_seq: 0,
|
||||
min_seq: 0,
|
||||
});
|
||||
let mut terminals = lock_displaced_terminals(registry);
|
||||
prune_completed_heal_statuses(&mut terminals);
|
||||
terminals.insert(request.id.clone(), Arc::clone(&terminal));
|
||||
terminal
|
||||
}
|
||||
|
||||
async fn remove_displaced_task_aliases(
|
||||
aliases: &Arc<Mutex<HashMap<String, HealTaskAlias>>>,
|
||||
terminals: &StdMutex<HashMap<String, Arc<CompletedHealStatus>>>,
|
||||
task_id: &str,
|
||||
terminal: &Arc<CompletedHealStatus>,
|
||||
) {
|
||||
let mut aliases = aliases.lock().await;
|
||||
let alias_ids = aliases
|
||||
.iter()
|
||||
.filter_map(|(alias_id, alias)| (alias.task_id == task_id).then_some(alias_id.clone()))
|
||||
.collect::<Vec<_>>();
|
||||
let mut displaced_terminals = lock_displaced_terminals(terminals);
|
||||
prune_completed_heal_statuses(&mut displaced_terminals);
|
||||
for alias_id in alias_ids {
|
||||
displaced_terminals.insert(alias_id, Arc::clone(terminal));
|
||||
}
|
||||
aliases.retain(|alias_id, alias| alias_id != task_id && alias.task_id != task_id);
|
||||
}
|
||||
|
||||
async fn remove_task_aliases_for_task(registry: &Arc<Mutex<HashMap<String, HealTaskAlias>>>, task_id: &str) {
|
||||
registry
|
||||
.lock()
|
||||
@@ -618,6 +672,14 @@ pub struct HealManager {
|
||||
/// are shared so the lookup helper can hand a completed entry to a
|
||||
/// caller without cloning the retained result window.
|
||||
completed_heals: Arc<Mutex<HashMap<String, Arc<CompletedHealStatus>>>>,
|
||||
/// Terminals for requests removed by priority displacement. An Accepted
|
||||
/// task ID remains queryable for the same process lifetime and the normal
|
||||
/// ten-minute status TTL; clients should treat `reason=displaced` as a
|
||||
/// terminal result and submit a fresh request. This sidecar is synchronous
|
||||
/// so admission can publish the terminal while the queue transition is
|
||||
/// still under its lock, without awaiting another tokio lock. Queue state
|
||||
/// is process-local, so this guarantee does not extend across restart.
|
||||
displaced_terminals: Arc<StdMutex<HashMap<String, Arc<CompletedHealStatus>>>>,
|
||||
/// Client tokens merged into an existing task id.
|
||||
task_aliases: Arc<Mutex<HashMap<String, HealTaskAlias>>>,
|
||||
/// Heal tasks waiting for a retry backoff to expire.
|
||||
@@ -659,6 +721,7 @@ struct HealQueueContext<'a> {
|
||||
heal_queue: &'a Arc<Mutex<PriorityHealQueue>>,
|
||||
active_heals: &'a Arc<Mutex<HashMap<String, Arc<HealTask>>>>,
|
||||
completed_heals: &'a Arc<Mutex<HashMap<String, Arc<CompletedHealStatus>>>>,
|
||||
displaced_terminals: &'a Arc<StdMutex<HashMap<String, Arc<CompletedHealStatus>>>>,
|
||||
task_aliases: &'a Arc<Mutex<HashMap<String, HealTaskAlias>>>,
|
||||
retrying_heals: &'a Arc<Mutex<HashMap<String, RetryingHeal>>>,
|
||||
mrf_repair_notice_targets: &'a Arc<StdMutex<HashMap<String, Vec<MrfRepairNoticeTarget>>>>,
|
||||
@@ -874,7 +937,7 @@ impl HealManager {
|
||||
result = "accepted_by_displacement",
|
||||
"Heal queue request accepted by displacement"
|
||||
});
|
||||
return HealAdmissionDecision::accepted_with_displacement(displaced.id);
|
||||
return HealAdmissionDecision::accepted_with_displacement(displaced);
|
||||
}
|
||||
|
||||
demote_to_debug_when!(per_object_request, warn, target: "rustfs::heal::manager", {
|
||||
@@ -1105,6 +1168,7 @@ impl HealManager {
|
||||
active_heals: Arc::new(Mutex::new(HashMap::new())),
|
||||
heal_queue: Arc::new(Mutex::new(PriorityHealQueue::new())),
|
||||
completed_heals: Arc::new(Mutex::new(HashMap::new())),
|
||||
displaced_terminals: Arc::new(StdMutex::new(HashMap::new())),
|
||||
task_aliases: Arc::new(Mutex::new(HashMap::new())),
|
||||
retrying_heals: Arc::new(Mutex::new(HashMap::new())),
|
||||
mrf_repair_notice_targets: Arc::new(StdMutex::new(HashMap::new())),
|
||||
@@ -1209,6 +1273,10 @@ impl HealManager {
|
||||
active_heals.clear();
|
||||
publish_active_heal_count(&active_heals);
|
||||
self.completed_heals.lock().await.clear();
|
||||
// Do not let the synchronous guard live across the following async lock.
|
||||
{
|
||||
lock_displaced_terminals(&self.displaced_terminals).clear();
|
||||
}
|
||||
self.task_aliases.lock().await.clear();
|
||||
self.retrying_heals.lock().await.clear();
|
||||
lock_mrf_repair_notice_targets(&self.mrf_repair_notice_targets).clear();
|
||||
@@ -1459,7 +1527,11 @@ impl HealManager {
|
||||
task_id = queued_id.to_owned();
|
||||
}
|
||||
let should_notify = matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
|
||||
let displaced_task_id = admission_decision.displaced_task_id;
|
||||
let displaced_task_id = admission_decision.displaced_task_id().map(ToOwned::to_owned);
|
||||
let displaced_terminal = admission_decision
|
||||
.displaced_request
|
||||
.as_ref()
|
||||
.map(|request| record_displaced_terminal(&self.displaced_terminals, request));
|
||||
if matches!(admission, HealAdmissionResult::Accepted | HealAdmissionResult::Merged)
|
||||
&& let Some(target) = mrf_notice_target
|
||||
{
|
||||
@@ -1473,8 +1545,12 @@ impl HealManager {
|
||||
drop(queue);
|
||||
drop(active_heals);
|
||||
|
||||
if let Some(displaced_task_id) = displaced_task_id {
|
||||
self.remove_aliases_for_task(&displaced_task_id).await;
|
||||
if let (Some(displaced_task_id), Some(displaced_terminal)) = (displaced_task_id, displaced_terminal) {
|
||||
// The queue has already removed the displaced request, so the
|
||||
// synchronous terminal sidecar was published before aliases and
|
||||
// MRF ownership are cleaned up.
|
||||
remove_displaced_task_aliases(&self.task_aliases, &self.displaced_terminals, &displaced_task_id, &displaced_terminal)
|
||||
.await;
|
||||
}
|
||||
|
||||
if should_notify {
|
||||
@@ -1549,6 +1625,15 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
|
||||
if terminal_completed.is_none() {
|
||||
let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals);
|
||||
prune_completed_heal_statuses(&mut displaced_terminals);
|
||||
terminal_completed = displaced_terminals
|
||||
.get(canonical_task_id)
|
||||
.filter(|terminal| matches_path(&terminal.heal_type))
|
||||
.cloned();
|
||||
}
|
||||
|
||||
match terminal_completed {
|
||||
Some(completed) => TaskStateLookup::Completed(completed),
|
||||
None => TaskStateLookup::NotFound,
|
||||
@@ -1669,9 +1754,19 @@ impl HealManager {
|
||||
|
||||
let mut completed_heals = self.completed_heals.lock().await;
|
||||
prune_completed_heal_statuses(&mut completed_heals);
|
||||
completed_heals
|
||||
if completed_heals
|
||||
.values()
|
||||
.any(|completed| heal_type_matches_path(&completed.heal_type, heal_path))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
drop(completed_heals);
|
||||
|
||||
let mut displaced_terminals = lock_displaced_terminals(&self.displaced_terminals);
|
||||
prune_completed_heal_statuses(&mut displaced_terminals);
|
||||
displaced_terminals
|
||||
.values()
|
||||
.any(|terminal| heal_type_matches_path(&terminal.heal_type, heal_path))
|
||||
}
|
||||
|
||||
/// Get task progress
|
||||
|
||||
@@ -21,6 +21,7 @@ impl HealManager {
|
||||
let heal_queue = self.heal_queue.clone();
|
||||
let active_heals = self.active_heals.clone();
|
||||
let task_aliases = self.task_aliases.clone();
|
||||
let displaced_terminals = self.displaced_terminals.clone();
|
||||
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
|
||||
let storage = self.storage.clone();
|
||||
let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
|
||||
@@ -481,6 +482,10 @@ impl HealManager {
|
||||
let admission = admission_decision.result;
|
||||
let should_notify =
|
||||
matches!(admission, HealAdmissionResult::Accepted) && config.event_driven_scheduler_enable;
|
||||
let displaced_terminal = admission_decision
|
||||
.displaced_request
|
||||
.as_ref()
|
||||
.map(|request| record_displaced_terminal(&displaced_terminals, request));
|
||||
if matches!(admission, HealAdmissionResult::Accepted)
|
||||
&& let Some(anchor) = recovery_anchor
|
||||
{
|
||||
@@ -491,8 +496,16 @@ impl HealManager {
|
||||
}
|
||||
drop(queue);
|
||||
drop(config);
|
||||
if let Some(displaced_task_id) = admission_decision.displaced_task_id {
|
||||
remove_task_aliases_for_task(&task_aliases, &displaced_task_id).await;
|
||||
if let (Some(displaced_task_id), Some(displaced_terminal)) =
|
||||
(admission_decision.displaced_task_id().map(ToOwned::to_owned), displaced_terminal)
|
||||
{
|
||||
remove_displaced_task_aliases(
|
||||
&task_aliases,
|
||||
&displaced_terminals,
|
||||
&displaced_task_id,
|
||||
&displaced_terminal,
|
||||
)
|
||||
.await;
|
||||
lock_mrf_repair_notice_targets(&mrf_repair_notice_targets).remove(&displaced_task_id);
|
||||
}
|
||||
if matches!(admission, HealAdmissionResult::Accepted) {
|
||||
|
||||
@@ -21,6 +21,7 @@ impl HealManager {
|
||||
let heal_queue = self.heal_queue.clone();
|
||||
let active_heals = self.active_heals.clone();
|
||||
let completed_heals = self.completed_heals.clone();
|
||||
let displaced_terminals = self.displaced_terminals.clone();
|
||||
let task_aliases = self.task_aliases.clone();
|
||||
let retrying_heals = self.retrying_heals.clone();
|
||||
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
|
||||
@@ -53,6 +54,7 @@ impl HealManager {
|
||||
heal_queue: &heal_queue,
|
||||
active_heals: &active_heals,
|
||||
completed_heals: &completed_heals,
|
||||
displaced_terminals: &displaced_terminals,
|
||||
task_aliases: &task_aliases,
|
||||
retrying_heals: &retrying_heals,
|
||||
mrf_repair_notice_targets: &mrf_repair_notice_targets,
|
||||
@@ -71,6 +73,7 @@ impl HealManager {
|
||||
heal_queue: &heal_queue,
|
||||
active_heals: &active_heals,
|
||||
completed_heals: &completed_heals,
|
||||
displaced_terminals: &displaced_terminals,
|
||||
task_aliases: &task_aliases,
|
||||
retrying_heals: &retrying_heals,
|
||||
mrf_repair_notice_targets: &mrf_repair_notice_targets,
|
||||
@@ -98,6 +101,7 @@ impl HealManager {
|
||||
heal_queue,
|
||||
active_heals,
|
||||
completed_heals,
|
||||
displaced_terminals,
|
||||
task_aliases,
|
||||
retrying_heals,
|
||||
mrf_repair_notice_targets,
|
||||
@@ -183,6 +187,7 @@ impl HealManager {
|
||||
let active_heals_clone = active_heals.clone();
|
||||
let heal_queue_clone = heal_queue.clone();
|
||||
let completed_heals_clone = completed_heals.clone();
|
||||
let displaced_terminals_clone = displaced_terminals.clone();
|
||||
let task_aliases_clone = task_aliases.clone();
|
||||
let retrying_heals_clone = retrying_heals.clone();
|
||||
let mrf_repair_notice_targets_clone = mrf_repair_notice_targets.clone();
|
||||
@@ -363,6 +368,7 @@ impl HealManager {
|
||||
let retry_heal_queue = heal_queue_clone.clone();
|
||||
let retrying_heals_for_spawn = retrying_heals_clone.clone();
|
||||
let retry_task_aliases = task_aliases_clone.clone();
|
||||
let retry_displaced_terminals = displaced_terminals_clone.clone();
|
||||
let retry_mrf_repair_notice_targets = mrf_repair_notice_targets_clone.clone();
|
||||
let retry_completed_heals = completed_heals_clone.clone();
|
||||
let retry_notify = notify_clone.clone();
|
||||
@@ -430,6 +436,14 @@ impl HealManager {
|
||||
let admission = admission_decision.result;
|
||||
let should_notify = matches!(admission, HealAdmissionResult::Accepted)
|
||||
&& retry_config.event_driven_scheduler_enable;
|
||||
// Publish the terminal synchronously while the
|
||||
// queue transition is protected. The subsequent
|
||||
// queue -> retrying handoff retains the lock order
|
||||
// used by operations_snapshot.
|
||||
let displaced_terminal = admission_decision
|
||||
.displaced_request
|
||||
.as_ref()
|
||||
.map(|request| record_displaced_terminal(&retry_displaced_terminals, request));
|
||||
match admission {
|
||||
HealAdmissionResult::Accepted => {
|
||||
// Transfer ownership while holding queue -> retrying,
|
||||
@@ -437,10 +451,18 @@ impl HealManager {
|
||||
#[cfg(test)]
|
||||
pause_retry_ownership_transition(&retry_request_id, true).await;
|
||||
retrying_heals_for_spawn.lock().await.remove(&retry_request_id);
|
||||
let displaced_task_id = admission_decision.displaced_task_id;
|
||||
let displaced_task_id = admission_decision.displaced_task_id().map(ToOwned::to_owned);
|
||||
drop(queue);
|
||||
if let Some(displaced_task_id) = displaced_task_id {
|
||||
remove_task_aliases_for_task(&retry_task_aliases, &displaced_task_id).await;
|
||||
if let (Some(displaced_task_id), Some(displaced_terminal)) =
|
||||
(displaced_task_id, displaced_terminal)
|
||||
{
|
||||
remove_displaced_task_aliases(
|
||||
&retry_task_aliases,
|
||||
&retry_displaced_terminals,
|
||||
&displaced_task_id,
|
||||
&displaced_terminal,
|
||||
)
|
||||
.await;
|
||||
remove_mrf_repair_notice_targets(
|
||||
&retry_mrf_repair_notice_targets,
|
||||
&displaced_task_id,
|
||||
|
||||
@@ -84,6 +84,7 @@ async fn process_manager_queue_once(manager: &HealManager) {
|
||||
heal_queue: &manager.heal_queue,
|
||||
active_heals: &manager.active_heals,
|
||||
completed_heals: &manager.completed_heals,
|
||||
displaced_terminals: &manager.displaced_terminals,
|
||||
task_aliases: &manager.task_aliases,
|
||||
retrying_heals: &manager.retrying_heals,
|
||||
mrf_repair_notice_targets: &manager.mrf_repair_notice_targets,
|
||||
@@ -2778,7 +2779,10 @@ async fn test_high_priority_request_displaces_lower_priority_when_queue_full() {
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
assert_eq!(manager.get_queue_length().await, 1);
|
||||
assert!(matches!(manager.get_task_status(&low_id).await, Err(Error::TaskNotFound { .. })));
|
||||
assert!(matches!(
|
||||
manager.get_task_status(&low_id).await,
|
||||
Ok(HealTaskStatus::Failed { error }) if error.contains("reason=displaced")
|
||||
));
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_task_status(&high_id)
|
||||
@@ -2788,6 +2792,263 @@ async fn test_high_priority_request_displaces_lower_priority_when_queue_full() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn displaced_task_remains_queryable() {
|
||||
let manager = HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
let mut displaced = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "displaced-bucket".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
);
|
||||
displaced.id = "displaced-task".to_string();
|
||||
let displaced_id = displaced.id.clone();
|
||||
manager
|
||||
.submit_heal_request(displaced)
|
||||
.await
|
||||
.expect("displaced request should queue");
|
||||
|
||||
let successor = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "successor-bucket".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
);
|
||||
manager
|
||||
.submit_heal_request(successor)
|
||||
.await
|
||||
.expect("successor should displace low work");
|
||||
|
||||
let report = manager
|
||||
.get_task_report(&displaced_id)
|
||||
.await
|
||||
.expect("displaced report should remain queryable");
|
||||
assert!(matches!(report.status, HealTaskStatus::Failed { ref error } if error.contains("reason=displaced")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn displaced_archive_failure_keeps_queryable_terminal() {
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "archive-failure".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
);
|
||||
request.id = "archive-failure-task".to_string();
|
||||
let request_id = request.id.clone();
|
||||
// The synchronous sidecar is the authoritative fallback when the normal
|
||||
// completed-task archive has no entry (the failure window that must not
|
||||
// turn an Accepted ID into NotFound).
|
||||
record_displaced_terminal(&manager.displaced_terminals, &request);
|
||||
assert!(manager.completed_heals.lock().await.is_empty());
|
||||
assert!(matches!(
|
||||
manager.get_task_status(&request_id).await,
|
||||
Ok(HealTaskStatus::Failed { error }) if error.contains("reason=displaced")
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scheduler_retry_displacement_keeps_evicted_task_queryable() {
|
||||
let manager = Arc::new(HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
event_driven_scheduler_enable: false,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
));
|
||||
let mut retry_request = HealRequest::object("retry-transition".to_string(), "object".to_string(), None);
|
||||
retry_request.priority = HealPriority::High;
|
||||
let retry_id = retry_request.id.clone();
|
||||
manager
|
||||
.submit_heal_request(retry_request)
|
||||
.await
|
||||
.expect("retry request should queue");
|
||||
|
||||
// Process exactly one queue cycle so the retry task is spawned without a
|
||||
// background scheduler consuming the filler request before the retry wakes.
|
||||
process_manager_queue_once(&manager).await;
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if manager.retrying_heals.lock().await.contains_key(&retry_id) {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retry request should enter backoff");
|
||||
|
||||
let filler = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "retry-displaced-filler".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
);
|
||||
let filler_id = filler.id.clone();
|
||||
manager
|
||||
.submit_heal_request(filler)
|
||||
.await
|
||||
.expect("filler request should occupy the queue");
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if matches!(
|
||||
manager.get_task_status(&filler_id).await,
|
||||
Ok(HealTaskStatus::Failed { ref error }) if error.contains("reason=displaced")
|
||||
) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retry admission should displace the filler request");
|
||||
assert_eq!(manager.get_queue_length().await, 1);
|
||||
assert_eq!(
|
||||
manager.get_task_status(&retry_id).await.expect("retry should be queued"),
|
||||
HealTaskStatus::Pending
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_displacers_produce_one_terminal_generation() {
|
||||
let manager = Arc::new(HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
));
|
||||
let mut displaced = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "concurrent-displaced".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
);
|
||||
displaced.id = "concurrent-displaced-task".to_string();
|
||||
let displaced_id = displaced.id.clone();
|
||||
manager
|
||||
.submit_heal_request(displaced)
|
||||
.await
|
||||
.expect("initial request should queue");
|
||||
|
||||
let first = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "concurrent-successor-a".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
);
|
||||
let second = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "concurrent-successor-b".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
);
|
||||
let (first_result, second_result) = tokio::join!(manager.submit_heal_request(first), manager.submit_heal_request(second));
|
||||
let accepted = [&first_result, &second_result]
|
||||
.into_iter()
|
||||
.filter(|result| matches!(result, Ok(HealAdmissionResult::Accepted)))
|
||||
.count();
|
||||
assert_eq!(accepted, 1, "exactly one concurrent displacer should win the full queue");
|
||||
assert!(
|
||||
first_result.is_ok() && second_result.is_ok(),
|
||||
"the losing request should receive a typed Full result"
|
||||
);
|
||||
let terminals = lock_displaced_terminals(&manager.displaced_terminals);
|
||||
assert_eq!(terminals.len(), 1);
|
||||
assert!(terminals.contains_key(&displaced_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn successor_chain_is_bounded_and_authorized() {
|
||||
let manager = HealManager::new(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
let mut original = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "authorized-original".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
);
|
||||
original.id = "authorized-original-task".to_string();
|
||||
let original_id = original.id.clone();
|
||||
manager.submit_heal_request(original).await.expect("original should queue");
|
||||
let mut duplicate = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "authorized-original".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
);
|
||||
duplicate.id = "authorized-duplicate-task".to_string();
|
||||
let duplicate_id = duplicate.id.clone();
|
||||
manager
|
||||
.submit_heal_request(duplicate)
|
||||
.await
|
||||
.expect("same-target duplicate should merge");
|
||||
let successor = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "authorized-successor".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::High,
|
||||
);
|
||||
let successor_id = successor.id.clone();
|
||||
manager.submit_heal_request(successor).await.expect("successor should queue");
|
||||
assert!(manager.task_aliases.lock().await.is_empty());
|
||||
assert!(matches!(manager.get_task_status(&original_id).await, Ok(HealTaskStatus::Failed { .. })));
|
||||
assert!(matches!(manager.get_task_status(&duplicate_id).await, Ok(HealTaskStatus::Failed { .. })));
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_task_status(&successor_id)
|
||||
.await
|
||||
.expect("successor should remain queued"),
|
||||
HealTaskStatus::Pending
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn displaced_terminal_expires_after_bounded_ttl() {
|
||||
let manager = HealManager::new(Arc::new(MockStorage), None);
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "expires".to_string(),
|
||||
},
|
||||
HealOptions::default(),
|
||||
HealPriority::Low,
|
||||
);
|
||||
request.id = "expires-task".to_string();
|
||||
let request_id = request.id.clone();
|
||||
record_displaced_terminal(&manager.displaced_terminals, &request);
|
||||
{
|
||||
let mut terminals = lock_displaced_terminals(&manager.displaced_terminals);
|
||||
let entry =
|
||||
Arc::get_mut(terminals.get_mut(&request_id).expect("terminal should be retained")).expect("test owns terminal entry");
|
||||
entry.completed_at = SystemTime::now() - KEEP_HEAL_TASK_STATUS_DURATION - Duration::from_secs(1);
|
||||
}
|
||||
assert!(matches!(manager.get_task_status(&request_id).await, Err(Error::TaskNotFound { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_displacing_registered_mrf_task_drops_notice_ownership() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
|
||||
@@ -54,6 +54,13 @@ pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_versio
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip";
|
||||
pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE: &str = "batch_read_version_request_encode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE: &str = "batch_read_version_request_decode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ: &str = "batch_read_version_disk_read";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch_read_version_response_json_encode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode";
|
||||
|
||||
const OPERATION_LABEL: &str = "operation";
|
||||
const BACKEND_LABEL: &str = "backend";
|
||||
|
||||
@@ -689,6 +689,14 @@ pub struct ScannerMetrics {
|
||||
pub cycle_max_objects: u64,
|
||||
#[serde(rename = "cycle_max_directories", default)]
|
||||
pub cycle_max_directories: u64,
|
||||
#[serde(rename = "cycle_timeout_total", default)]
|
||||
pub cycle_timeout_total: u64,
|
||||
#[serde(rename = "cycle_recovery_required_total", default)]
|
||||
pub cycle_recovery_required_total: u64,
|
||||
#[serde(rename = "cycle_last_progress_age", default)]
|
||||
pub cycle_last_progress_age: u64,
|
||||
#[serde(rename = "leader_lease_without_progress", default)]
|
||||
pub leader_lease_without_progress: bool,
|
||||
#[serde(rename = "bitrot_cycle_enabled", default)]
|
||||
pub bitrot_cycle_enabled: bool,
|
||||
#[serde(rename = "bitrot_cycle_seconds", default)]
|
||||
@@ -764,6 +772,8 @@ impl ScannerMetrics {
|
||||
self.cycle_max_duration_seconds = other.cycle_max_duration_seconds;
|
||||
self.cycle_max_objects = other.cycle_max_objects;
|
||||
self.cycle_max_directories = other.cycle_max_directories;
|
||||
self.cycle_last_progress_age = other.cycle_last_progress_age;
|
||||
self.leader_lease_without_progress = other.leader_lease_without_progress;
|
||||
self.bitrot_cycle_enabled = other.bitrot_cycle_enabled;
|
||||
self.bitrot_cycle_seconds = other.bitrot_cycle_seconds;
|
||||
}
|
||||
@@ -857,6 +867,12 @@ impl ScannerMetrics {
|
||||
.saturating_add(other.last_cycle_replication_checks);
|
||||
self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves);
|
||||
self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles);
|
||||
self.cycle_timeout_total = self.cycle_timeout_total.saturating_add(other.cycle_timeout_total);
|
||||
self.cycle_recovery_required_total = self
|
||||
.cycle_recovery_required_total
|
||||
.saturating_add(other.cycle_recovery_required_total);
|
||||
self.cycle_last_progress_age = self.cycle_last_progress_age.max(other.cycle_last_progress_age);
|
||||
self.leader_lease_without_progress |= other.leader_lease_without_progress;
|
||||
self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles);
|
||||
self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown);
|
||||
self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime);
|
||||
|
||||
@@ -195,7 +195,8 @@ pub struct BucketPolicyArgs<'a> {
|
||||
#[derive(Serialize, Deserialize, Clone, Default, Debug)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BucketPolicy {
|
||||
#[serde(default, rename = "Id", skip_serializing_if = "ID::is_empty")]
|
||||
// RUSTFS_COMPAT_TODO(rustfs-6339): accept bucket policies persisted with the legacy "ID" key. Remove after migration tooling rewrites every retained legacy bucket policy.
|
||||
#[serde(default, rename = "Id", alias = "ID", skip_serializing_if = "ID::is_empty")]
|
||||
pub id: ID,
|
||||
#[serde(rename = "Version")]
|
||||
pub version: String,
|
||||
@@ -2786,7 +2787,7 @@ mod test {
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse");
|
||||
|
||||
// Verify empty fields are omitted
|
||||
assert!(!parsed.as_object().unwrap().contains_key("ID"), "Empty ID should be omitted");
|
||||
assert!(parsed.get("Id").is_none(), "Empty ID should be omitted");
|
||||
|
||||
let statement = &parsed["Statement"][0];
|
||||
assert!(!statement.as_object().unwrap().contains_key("Sid"), "Empty Sid should be omitted");
|
||||
@@ -2809,6 +2810,43 @@ mod test {
|
||||
assert_eq!(statement["Principal"]["AWS"], "*");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_policy_deserializes_legacy_id() {
|
||||
let legacy_policy = br#"{"ID":"","Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"NotAction":[],"Resource":["arn:aws:s3:::bucket/*"],"NotResource":[],"Condition":{}}]}"#;
|
||||
|
||||
let policy: BucketPolicy =
|
||||
serde_json::from_slice(legacy_policy).expect("bucket policy with legacy ID should deserialize");
|
||||
assert!(policy.id.is_empty());
|
||||
policy.is_valid().expect("legacy bucket policy should remain valid");
|
||||
|
||||
let policy: BucketPolicy = serde_json::from_str(r#"{"ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#)
|
||||
.expect("non-empty legacy ID should deserialize");
|
||||
assert_eq!(policy.id.0, "legacy-policy");
|
||||
|
||||
let serialized = serde_json::to_value(&policy).expect("bucket policy should serialize");
|
||||
assert_eq!(serialized["Id"], "legacy-policy");
|
||||
assert!(serialized.get("ID").is_none(), "legacy ID spelling should not be serialized");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_policy_legacy_id_alias_remains_strict() {
|
||||
let unknown_field = r#"{"Version":"2012-10-17","Statement":[],"Unexpected":true}"#;
|
||||
let error =
|
||||
serde_json::from_str::<BucketPolicy>(unknown_field).expect_err("unrelated unknown fields should remain rejected");
|
||||
assert!(
|
||||
error.to_string().contains("unknown field `Unexpected`"),
|
||||
"unexpected deserialization error: {error}"
|
||||
);
|
||||
|
||||
let duplicate_id = r#"{"Id":"current-policy","ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#;
|
||||
let error = serde_json::from_str::<BucketPolicy>(duplicate_id)
|
||||
.expect_err("canonical and legacy ID fields should not be accepted together");
|
||||
assert!(
|
||||
error.to_string().contains("duplicate field `Id`"),
|
||||
"unexpected deserialization error: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_existing_object_tag_condition_helpers() {
|
||||
let identity_policy = Policy::parse_config(
|
||||
|
||||
@@ -722,6 +722,10 @@ pub struct DeleteVersionsResponse {
|
||||
pub errors: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
|
||||
#[prost(message, optional, tag = "3")]
|
||||
pub error: ::core::option::Option<Error>,
|
||||
/// Senders dual-write the legacy strings and typed entries. Receivers prefer typed entries
|
||||
/// when present and fall back to strings for peers that predate this field. Code zero means success.
|
||||
#[prost(message, repeated, tag = "4")]
|
||||
pub item_errors: ::prost::alloc::vec::Vec<Error>,
|
||||
}
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct ReadMultipleRequest {
|
||||
|
||||
@@ -2106,6 +2106,9 @@ pub enum ChannelClass {
|
||||
Bulk,
|
||||
}
|
||||
|
||||
// Keep multiplexed unary RPCs below h2's per-connection small-frame budget.
|
||||
const INTERNODE_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
|
||||
/// Whether control/bulk channel isolation is enabled (env-gated, default off for safe rollout).
|
||||
fn channel_isolation_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(
|
||||
@@ -2188,6 +2191,7 @@ async fn build_channel(dial_addr: &str, cache_key: &str) -> Result<Channel, Box<
|
||||
let mut connector = Endpoint::from_shared(dial_addr.to_string())?
|
||||
// Fast connection timeout for dead peer detection
|
||||
.connect_timeout(connect_timeout)
|
||||
.concurrency_limit(INTERNODE_RPC_CONCURRENCY_LIMIT)
|
||||
// TCP-level keepalive - OS will probe connection
|
||||
.tcp_keepalive(Some(tcp_keepalive))
|
||||
// Disable Nagle so latency-sensitive control-plane RPCs (locks/health) are not batched
|
||||
|
||||
@@ -493,6 +493,9 @@ message DeleteVersionsResponse {
|
||||
bool success = 1;
|
||||
repeated string errors = 2;
|
||||
optional Error error = 3;
|
||||
// Senders dual-write the legacy strings and typed entries. Receivers prefer typed entries
|
||||
// when present and fall back to strings for peers that predate this field. Code zero means success.
|
||||
repeated Error item_errors = 4;
|
||||
}
|
||||
|
||||
message ReadMultipleRequest {
|
||||
|
||||
@@ -60,7 +60,9 @@ pub const REPLICATION_READ_ONLY_HISTORICAL_FIELDS: &[&str] = &[
|
||||
"Destination.ReplicationTime",
|
||||
];
|
||||
|
||||
pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 1;
|
||||
// v2: disableProxy moved from unsupported to writable (per-target read-proxy
|
||||
// opt-out is accepted by set-remote-target and the `proxy` update op).
|
||||
pub const REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION: u32 = 2;
|
||||
|
||||
pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[
|
||||
"sourcebucket",
|
||||
@@ -83,9 +85,12 @@ pub const REMOTE_TARGET_WRITABLE_FIELDS: &[&str] = &[
|
||||
// madmin default of 60s); the per-target health-check interval is not
|
||||
// yet applied — the heartbeat keeps its global env-configured interval.
|
||||
"healthCheckDuration",
|
||||
// Per-target read-proxy opt-out, consumed by the proxy-target selector
|
||||
// (contract v2; previously only importable via MinIO bucket-targets.json).
|
||||
"disableProxy",
|
||||
];
|
||||
|
||||
pub const REMOTE_TARGET_UNSUPPORTED_FIELDS: &[&str] = &["disableProxy", "edge", "edgeSyncBeforeExpiry"];
|
||||
pub const REMOTE_TARGET_UNSUPPORTED_FIELDS: &[&str] = &["edge", "edgeSyncBeforeExpiry"];
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ObjectOpts {
|
||||
|
||||
@@ -125,6 +125,34 @@ pub(crate) async fn read_config_with_revision<S: ScannerObjectIO>(
|
||||
}
|
||||
}
|
||||
|
||||
/// Read only the object revision without materializing its body.
|
||||
pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
|
||||
match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(reader) => reader
|
||||
.object_info
|
||||
.etag
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.map(DataUsageCacheRevision::Etag)
|
||||
.ok_or_else(|| StorageError::other(format!("scanner config object {path} has no ETag"))),
|
||||
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
|
||||
Ok(DataUsageCacheRevision::Missing)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision,
|
||||
@@ -146,6 +174,11 @@ pub static LEGACY_DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
|
||||
pub static DATA_USAGE_BLOOM_NAME_PATH: LazyLock<String> =
|
||||
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_BLOOM_NAME}"));
|
||||
|
||||
/// Durable companion object for a cycle-state object which cannot be decoded.
|
||||
/// The primary object is deliberately never replaced or deleted by recovery.
|
||||
pub static DATA_USAGE_BLOOM_RECOVERY_PATH: LazyLock<String> =
|
||||
LazyLock::new(|| format!("{}.recovery-required.json", DATA_USAGE_BLOOM_NAME_PATH.as_str()));
|
||||
|
||||
pub static BACKGROUND_HEAL_INFO_PATH: LazyLock<String> =
|
||||
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}.background-heal.json"));
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ impl DataUsageCache {
|
||||
let loaded = Self::load_cache(store.clone(), name).await?;
|
||||
let backup = match loaded.backup_revision {
|
||||
Some(revision) => Some(revision),
|
||||
None => match Self::revision_for_path(store, &backup_path).await {
|
||||
None => match read_config_revision(store, &backup_path).await {
|
||||
Ok(revision) => Some(revision),
|
||||
Err(err) => {
|
||||
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
|
||||
@@ -336,33 +336,6 @@ impl DataUsageCache {
|
||||
}
|
||||
}
|
||||
|
||||
async fn revision_for_path<S: ScannerObjectIO>(store: Arc<S>, path: &str) -> StorageResult<DataUsageCacheRevision> {
|
||||
match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(reader) => reader
|
||||
.object_info
|
||||
.etag
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.map(DataUsageCacheRevision::Etag)
|
||||
.ok_or_else(|| StorageError::other(format!("scanner cache object {path} has no ETag"))),
|
||||
Err(Error::FileNotFound | Error::VolumeNotFound | Error::ObjectNotFound(_, _) | Error::BucketNotFound(_)) => {
|
||||
Ok(DataUsageCacheRevision::Missing)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn cache_save_timeout() -> Duration {
|
||||
crate::runtime_config::scanner_cache_save_timeout()
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::persistence::DataUsageCacheLoadAttempt;
|
||||
use super::*;
|
||||
use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO};
|
||||
use crate::{ScannerGetObjectReader, ScannerPutObjReader};
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats};
|
||||
use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
|
||||
use serde_json::Value;
|
||||
use std::io::Cursor;
|
||||
use std::pin::Pin;
|
||||
@@ -1636,7 +1636,7 @@ fn size_recursive_prunes_empty_and_preserves_threshold_replication_stats() {
|
||||
replication_stats: Some(ReplicationAllStats {
|
||||
targets: HashMap::from([(
|
||||
"arn:test:threshold".to_string(),
|
||||
ReplicationStats {
|
||||
ReplicationTargetUsage {
|
||||
after_threshold_count: 1,
|
||||
..Default::default()
|
||||
},
|
||||
|
||||
@@ -75,7 +75,10 @@ pub use remote_scanner::{
|
||||
};
|
||||
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
|
||||
pub use rustfs_common::last_minute;
|
||||
pub use scanner::{ScannerCycleScheduleStatus, init_data_scanner, scanner_cycle_schedule_status, scanner_topology_digest};
|
||||
pub use scanner::{
|
||||
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner,
|
||||
reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest,
|
||||
};
|
||||
pub use scanner_io::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket,
|
||||
record_dirty_usage_bucket, record_scanner_maintenance_change, scanner_activity_epoch, scanner_dirty_usage_state,
|
||||
|
||||
@@ -125,7 +125,10 @@ impl Default for ScannerRuntimeConfig {
|
||||
cycle_interval_source: ScannerRuntimeConfigSource::Default,
|
||||
bitrot_cycle: Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS)),
|
||||
bitrot_cycle_source: ScannerRuntimeConfigSource::Default,
|
||||
cycle_budget: ScannerCycleBudgetConfig::default(),
|
||||
cycle_budget: ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
|
||||
..Default::default()
|
||||
},
|
||||
cycle_max_duration_source: ScannerRuntimeConfigSource::Default,
|
||||
cycle_max_objects_source: ScannerRuntimeConfigSource::Default,
|
||||
cycle_max_directories_source: ScannerRuntimeConfigSource::Default,
|
||||
@@ -374,7 +377,10 @@ fn validate_persisted_scanner_runtime_config(config: &ServerConfig) -> Result<()
|
||||
}
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_START_DELAY, "")?;
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE, "")?;
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)?;
|
||||
if let Some(value) = config_value(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) {
|
||||
let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?;
|
||||
cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)?;
|
||||
}
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_OBJECTS, DEFAULT_SCANNER_CYCLE_MAX_OBJECTS)?;
|
||||
validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DIRECTORIES, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES)?;
|
||||
if let Some(value) = config_value(heal_kvs, HEAL_BITROT_CYCLE, DEFAULT_HEAL_BITROT_CYCLE_SECS) {
|
||||
@@ -436,19 +442,46 @@ fn lookup_max_wait(
|
||||
Ok((speed.max_sleep(), speed_source))
|
||||
}
|
||||
|
||||
fn lookup_optional_seconds(
|
||||
kvs: Option<&KVS>,
|
||||
key: &'static str,
|
||||
env_key: &'static str,
|
||||
default: u64,
|
||||
) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
|
||||
if let Some(secs) = rustfs_utils::get_env_opt_u64(env_key) {
|
||||
return Ok((Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Env));
|
||||
fn lookup_cycle_duration(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
|
||||
match rustfs_utils::get_env_parse_outcome::<u64>(ENV_SCANNER_CYCLE_MAX_DURATION_SECS) {
|
||||
rustfs_utils::EnvParseOutcome::Parsed(secs) => {
|
||||
return cycle_duration_from_secs(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, secs)
|
||||
.map(|duration| (duration, ScannerRuntimeConfigSource::Env));
|
||||
}
|
||||
rustfs_utils::EnvParseOutcome::Invalid => {
|
||||
// Do not include the raw environment value in the typed error:
|
||||
// deployments occasionally put sensitive material in inherited
|
||||
// environment snapshots. The key still identifies the control.
|
||||
return Err(invalid_value(
|
||||
ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
|
||||
"<invalid>",
|
||||
"expected unsigned integer seconds",
|
||||
));
|
||||
}
|
||||
rustfs_utils::EnvParseOutcome::Absent => {}
|
||||
}
|
||||
if let Some(value) = config_value(kvs, key, default) {
|
||||
return parse_config_u64(key, value).map(|secs| (Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Config));
|
||||
|
||||
if let Some(value) = config_value(kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) {
|
||||
let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?;
|
||||
return cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)
|
||||
.map(|duration| (duration, ScannerRuntimeConfigSource::Config));
|
||||
}
|
||||
Ok((None, ScannerRuntimeConfigSource::Default))
|
||||
|
||||
Ok((
|
||||
Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)),
|
||||
ScannerRuntimeConfigSource::Default,
|
||||
))
|
||||
}
|
||||
|
||||
fn cycle_duration_from_secs(key: &'static str, secs: u64) -> Result<Option<Duration>, ScannerRuntimeConfigError> {
|
||||
if secs == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let duration = Duration::from_secs(secs);
|
||||
if std::time::Instant::now().checked_add(duration).is_none() {
|
||||
return Err(invalid_value(key, "<overflow>", "duration exceeds the timer range"));
|
||||
}
|
||||
Ok(Some(duration))
|
||||
}
|
||||
|
||||
fn lookup_start_delay(kvs: Option<&KVS>) -> Result<(Option<Duration>, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
|
||||
@@ -553,12 +586,7 @@ pub(crate) fn lookup_scanner_runtime_config(
|
||||
(speed.cycle_interval(), speed_source)
|
||||
};
|
||||
|
||||
let (cycle_max_duration, cycle_max_duration_source) = lookup_optional_seconds(
|
||||
scanner_kvs,
|
||||
SCANNER_CYCLE_MAX_DURATION,
|
||||
ENV_SCANNER_CYCLE_MAX_DURATION_SECS,
|
||||
DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS,
|
||||
)?;
|
||||
let (cycle_max_duration, cycle_max_duration_source) = lookup_cycle_duration(scanner_kvs)?;
|
||||
let (cycle_max_objects, cycle_max_objects_source) = lookup_count_budget(
|
||||
scanner_kvs,
|
||||
SCANNER_CYCLE_MAX_OBJECTS,
|
||||
@@ -863,10 +891,10 @@ mod tests {
|
||||
use rustfs_config::server_config::{Config as ServerConfig, KVS};
|
||||
use rustfs_config::{
|
||||
DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS,
|
||||
ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED,
|
||||
HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
|
||||
SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
||||
ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY,
|
||||
ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE,
|
||||
SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION,
|
||||
SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
@@ -941,6 +969,50 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_unset_budget_uses_safe_default_but_explicit_zero_is_unbounded() {
|
||||
let config = server_config_with_scanner(&[]);
|
||||
with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
|
||||
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||
assert_eq!(resolved.cycle_budget.max_duration, Some(Duration::from_secs(1800)));
|
||||
assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Default);
|
||||
});
|
||||
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "0")]);
|
||||
with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || {
|
||||
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||
assert_eq!(resolved.cycle_budget.max_duration, None);
|
||||
assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Config);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cycle_budget_invalid_or_overflow_config_is_rejected() {
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("invalid"), || {
|
||||
let error = lookup_scanner_runtime_config(None).expect_err("invalid duration env must be rejected");
|
||||
assert!(error.to_string().contains(ENV_SCANNER_CYCLE_MAX_DURATION_SECS));
|
||||
assert!(error.to_string().contains("<invalid>"));
|
||||
assert!(!error.to_string().contains(": invalid ("));
|
||||
});
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551616"), || {
|
||||
assert!(lookup_scanner_runtime_config(None).is_err());
|
||||
});
|
||||
with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551615"), || {
|
||||
assert!(lookup_scanner_runtime_config(None).is_err());
|
||||
});
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "not-a-duration")]);
|
||||
assert!(lookup_scanner_runtime_config(Some(&config)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_runtime_config_validation_rejects_overflow_persisted_duration() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "18446744073709551615")]);
|
||||
|
||||
let error = validate_scanner_runtime_config(&config)
|
||||
.expect_err("persisted duration that exceeds the timer range must be rejected");
|
||||
assert!(error.to_string().contains(SCANNER_CYCLE_MAX_DURATION));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_runtime_config_normalizes_persisted_default_speed() {
|
||||
let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]);
|
||||
|
||||
+281
-63
@@ -20,7 +20,7 @@ use std::sync::{Arc, LazyLock, RwLock};
|
||||
|
||||
use crate::data_usage_define::{
|
||||
BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH,
|
||||
DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_with_revision,
|
||||
DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision,
|
||||
};
|
||||
use crate::runtime_config::{
|
||||
ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle,
|
||||
@@ -52,11 +52,10 @@ use rustfs_config::{
|
||||
};
|
||||
use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
|
||||
use rustfs_data_usage::observed_data_usage_is_newer;
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
#[cfg(test)]
|
||||
use tokio::sync::Notify;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::{Notify, mpsc};
|
||||
use tokio::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::AbortOnDropHandle;
|
||||
@@ -104,6 +103,13 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
|
||||
/// unavailable peer cannot drive a tight retry loop.
|
||||
const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
/// A transient backend outage remains self-healing after the short retry
|
||||
/// budget is exhausted, but the probe is intentionally sparse until storage
|
||||
/// recovers or an operator reset wakes the scanner.
|
||||
const SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
||||
/// Permanent recovery states still get a sparse status probe so a reset that
|
||||
/// races the wait registration cannot leave the scanner asleep forever.
|
||||
const SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL: Duration = Duration::from_secs(5 * 60);
|
||||
const SCANNER_LEADER_LOCK_POLL_INTERVAL: Duration = Duration::from_secs(1);
|
||||
#[cfg(not(test))]
|
||||
const SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
@@ -125,6 +131,12 @@ type ScannerCycleStatePersistTestHook = (u64, Arc<Notify>);
|
||||
static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCycleStatePersistTestHook>>> =
|
||||
LazyLock::new(|| StdMutex::new(None));
|
||||
|
||||
static SCANNER_CYCLE_RECOVERY_WAKE: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
|
||||
pub(super) fn notify_scanner_cycle_recovery_wake() {
|
||||
SCANNER_CYCLE_RECOVERY_WAKE.notify_one();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct ScannerCycleStatePersistTestHookGuard;
|
||||
|
||||
@@ -576,19 +588,21 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
tokio::time::sleep(sleep_time).await;
|
||||
}
|
||||
|
||||
let mut transient_backoff = ScannerRetryBackoff::default();
|
||||
let mut recovery_retry_count = 0_u32;
|
||||
loop {
|
||||
if ctx_clone.is_cancelled() {
|
||||
break;
|
||||
}
|
||||
|
||||
if let Err(e) = run_data_scanner_with_maintenance_state(
|
||||
let run_result = run_data_scanner_with_maintenance_state(
|
||||
ctx_clone.clone(),
|
||||
storeapi_clone.clone(),
|
||||
startup_features,
|
||||
startup_maintenance_generation,
|
||||
)
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
if let Err(e) = &run_result {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
@@ -599,11 +613,52 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc<ECStore>) {
|
||||
"Scanner runtime iteration failed"
|
||||
);
|
||||
}
|
||||
let recovery_status = scanner_cycle_recovery_status();
|
||||
if recovery_status.retryable {
|
||||
recovery_retry_count = recovery_retry_count.saturating_add(1);
|
||||
let _ = record_scanner_cycle_recovery_retry(recovery_retry_count);
|
||||
} else {
|
||||
recovery_retry_count = 0;
|
||||
}
|
||||
|
||||
let recovery_status = scanner_cycle_recovery_status();
|
||||
if recovery_status.state == "paused" {
|
||||
transient_backoff.record_retryable_cycle(false);
|
||||
tokio::select! {
|
||||
_ = ctx_clone.cancelled() => break,
|
||||
_ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
|
||||
_ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {},
|
||||
}
|
||||
recovery_retry_count = 0;
|
||||
continue;
|
||||
}
|
||||
if !recovery_status.retryable
|
||||
&& matches!(recovery_status.state.as_str(), "blocked" | "recovery-required" | "cleanup-pending")
|
||||
{
|
||||
transient_backoff.record_retryable_cycle(false);
|
||||
tokio::select! {
|
||||
_ = ctx_clone.cancelled() => break,
|
||||
_ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
|
||||
_ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_BLOCKED_PROBE_INTERVAL) => {},
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let retry_delay = if recovery_status.retryable || run_result.is_err() {
|
||||
transient_backoff.record_retryable_cycle(true);
|
||||
transient_backoff
|
||||
.retry_interval(scanner_cycle_interval())
|
||||
.unwrap_or(SCANNER_RETRY_BASE_INTERVAL)
|
||||
} else {
|
||||
transient_backoff.record_retryable_cycle(false);
|
||||
randomized_cycle_delay()
|
||||
};
|
||||
// Backoff before retrying after lock contention or scanner-level failures.
|
||||
// Keep this cancellation-aware so shutdown is not delayed by backoff sleep.
|
||||
tokio::select! {
|
||||
_ = ctx_clone.cancelled() => break,
|
||||
_ = tokio::time::sleep(randomized_cycle_delay()) => {}
|
||||
_ = SCANNER_CYCLE_RECOVERY_WAKE.notified() => {},
|
||||
_ = tokio::time::sleep(retry_delay) => {}
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -983,20 +1038,116 @@ fn data_usage_persist_timeout() -> Duration {
|
||||
DataUsageCache::persistence_timeout()
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
#[cfg(test)]
|
||||
const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_millis(50);
|
||||
|
||||
async fn fence_scanner_epoch_after_cycle_timeout<Store, LockLost>(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<Store>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
cycle_revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: &mut u64,
|
||||
lock_lost: LockLost,
|
||||
) -> bool
|
||||
where
|
||||
Store: ScannerObjectIO,
|
||||
LockLost: Future<Output = ()>,
|
||||
{
|
||||
let fence_ctx = ctx.child_token();
|
||||
let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch);
|
||||
tokio::pin!(claim);
|
||||
tokio::pin!(lock_lost);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = &mut lock_lost => {
|
||||
fence_ctx.cancel();
|
||||
false
|
||||
}
|
||||
result = tokio::time::timeout(SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT, &mut claim) => {
|
||||
result.unwrap_or(false) && !fence_ctx.is_cancelled()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ScannerCycleDeadlineState<'a> {
|
||||
cycle_info: &'a mut CurrentCycle,
|
||||
cycle_revision: &'a mut DataUsageCacheRevision,
|
||||
leader_epoch: &'a mut u64,
|
||||
cycle_budget: &'a ScannerCycleBudget,
|
||||
}
|
||||
|
||||
fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool {
|
||||
!worker_stopped || !cycle_state_persisted || !generation_fenced
|
||||
}
|
||||
|
||||
async fn handle_scanner_cycle_deadline<Store>(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: Arc<Store>,
|
||||
state: ScannerCycleDeadlineState<'_>,
|
||||
worker_stopped: bool,
|
||||
guard: &mut NamespaceLockGuard,
|
||||
) where
|
||||
Store: ScannerObjectIO,
|
||||
{
|
||||
let fenced = fence_scanner_epoch_after_cycle_timeout(
|
||||
ctx,
|
||||
storeapi,
|
||||
state.cycle_info,
|
||||
state.cycle_revision,
|
||||
state.leader_epoch,
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
.await;
|
||||
let cycle_state_persisted = state.cycle_budget.cycle_state_persisted();
|
||||
let recovery_required = cycle_timeout_requires_recovery(worker_stopped, cycle_state_persisted, fenced);
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "cycle_timeout",
|
||||
worker_stopped,
|
||||
cycle_state_persisted,
|
||||
generation_fenced = fenced,
|
||||
recovery_required,
|
||||
"Scanner cycle deadline expired; durable cursor/generation fencing completed when possible"
|
||||
);
|
||||
global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
|
||||
// Stop renewing before releasing the lease. A new leader can then claim the
|
||||
// higher persisted generation instead of inheriting the expired worker.
|
||||
guard.release();
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
|
||||
cycle_info.current = 0;
|
||||
global_metrics().clear_current_scan_mode();
|
||||
cycle_metrics_guard.finish(cycle_info.clone()).await;
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
#[hotpath::measure]
|
||||
#[cfg(test)]
|
||||
async fn run_data_scanner_cycle(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: &Arc<ECStore>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
cycle_revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
) -> ScannerCycleOutcome {
|
||||
let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config());
|
||||
run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget).await
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
#[hotpath::measure]
|
||||
async fn run_data_scanner_cycle_with_budget(
|
||||
ctx: &CancellationToken,
|
||||
storeapi: &Arc<ECStore>,
|
||||
cycle_info: &mut CurrentCycle,
|
||||
cycle_revision: &mut DataUsageCacheRevision,
|
||||
leader_epoch: u64,
|
||||
cycle_budget: Arc<ScannerCycleBudget>,
|
||||
) -> ScannerCycleOutcome {
|
||||
let _activity_guard = ScannerActivityGuard::new();
|
||||
if let Err(err) = refresh_scanner_runtime_config_from_global() {
|
||||
@@ -1012,7 +1163,11 @@ async fn run_data_scanner_cycle(
|
||||
}
|
||||
let configured_cycle_interval = scanner_cycle_interval();
|
||||
let configured_bitrot_cycle = scanner_bitrot_cycle();
|
||||
let cycle_budget_config = scanner_cycle_budget_config();
|
||||
let cycle_budget_config = ScannerCycleBudgetConfig {
|
||||
max_duration: cycle_budget.max_duration(),
|
||||
max_objects: cycle_budget.max_objects(),
|
||||
max_directories: cycle_budget.max_directories(),
|
||||
};
|
||||
let usage_persist_timeout = data_usage_persist_timeout();
|
||||
global_metrics().record_scanner_cycle_config(
|
||||
configured_cycle_interval,
|
||||
@@ -1083,7 +1238,6 @@ async fn run_data_scanner_cycle(
|
||||
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
|
||||
|
||||
let done_cycle = Metrics::time(Metric::ScanCycle);
|
||||
let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
|
||||
let scan_result = storeapi
|
||||
.clone()
|
||||
.nsscanner_with_status(
|
||||
@@ -1223,7 +1377,7 @@ async fn run_data_scanner_cycle(
|
||||
"Scanner cycle is recovering to a newer durable cache generation"
|
||||
);
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
return if persist_required_scanner_cycle_floor(
|
||||
let persisted = persist_required_scanner_cycle_floor(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
@@ -1232,8 +1386,9 @@ async fn run_data_scanner_cycle(
|
||||
required_cycle,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
@@ -1291,7 +1446,7 @@ async fn run_data_scanner_cycle(
|
||||
scan_cycle_partial_reason(budget_reason),
|
||||
scan_cycle_partial_source(budget_reason),
|
||||
);
|
||||
return if finalize_partial_scan_cycle(
|
||||
let persisted = finalize_partial_scan_cycle(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
@@ -1299,8 +1454,9 @@ async fn run_data_scanner_cycle(
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
@@ -1375,7 +1531,7 @@ async fn run_data_scanner_cycle(
|
||||
);
|
||||
}
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
return if finalize_partial_scan_cycle(
|
||||
let persisted = finalize_partial_scan_cycle(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
cycle_info,
|
||||
@@ -1383,8 +1539,9 @@ async fn run_data_scanner_cycle(
|
||||
leader_epoch,
|
||||
&mut cycle_metrics_guard,
|
||||
)
|
||||
.await
|
||||
{
|
||||
.await;
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
} else {
|
||||
ScannerCycleOutcome::Failed
|
||||
@@ -1425,6 +1582,7 @@ async fn run_data_scanner_cycle(
|
||||
)
|
||||
.await
|
||||
{
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
emit_scan_cycle_superseded(cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Superseded;
|
||||
}
|
||||
@@ -1457,6 +1615,7 @@ async fn run_data_scanner_cycle(
|
||||
emit_scan_cycle_complete(false, cycle_start.elapsed());
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
|
||||
done_cycle();
|
||||
emit_scan_cycle_complete(true, cycle_start.elapsed());
|
||||
@@ -1521,7 +1680,7 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
) -> Result<(), ScannerError> {
|
||||
reset_scanner_cycle_schedule();
|
||||
// Acquire leader lock (write lock) to ensure only one scanner runs
|
||||
let guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
|
||||
let mut guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await {
|
||||
Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await {
|
||||
Ok(guard) => {
|
||||
record_scanner_leader_lock_state("acquired");
|
||||
@@ -1606,40 +1765,22 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
observe_scanner_activity(&storeapi, distributed, &mut scanner_activity_seen).await;
|
||||
}
|
||||
|
||||
let (buf, mut cycle_revision) = match read_config_with_revision(storeapi.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str()).await {
|
||||
Ok((buf, revision)) => (buf.unwrap_or_default(), revision),
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
|
||||
state = "revision_load_failed",
|
||||
error = %err,
|
||||
"Scanner cycle state revision load failed"
|
||||
);
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let (mut cycle_info, mut leader_epoch) = match decode_scanner_cycle_state_for_startup(&buf) {
|
||||
Ok(state) => state,
|
||||
Err(err) => {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_PERSIST_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
path = %&*DATA_USAGE_BLOOM_NAME_PATH,
|
||||
state = "cycle_decode_failed",
|
||||
error = %err,
|
||||
"Scanner stopped because persisted cycle state is invalid"
|
||||
);
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
let (mut cycle_info, mut leader_epoch, mut cycle_revision) =
|
||||
match load_scanner_cycle_state_for_startup(storeapi.clone()).await {
|
||||
ScannerCycleStateStartup::Ready {
|
||||
cycle,
|
||||
leader_epoch,
|
||||
revision,
|
||||
} => (cycle, leader_epoch, revision),
|
||||
ScannerCycleStateStartup::Blocked => {
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleStateStartup::Transient(err) => {
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let usage_floor = match persisted_usage_floor(storeapi.clone()).await {
|
||||
Ok(floor) => floor,
|
||||
Err(err) => {
|
||||
@@ -1704,13 +1845,49 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
return Ok(());
|
||||
}
|
||||
let cycle_ctx = ctx.child_token();
|
||||
let initial_outcome = await_scanner_cycle_with_lock_fence(
|
||||
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
|
||||
let initial_outcome = match await_scanner_cycle_with_budget_fence(
|
||||
&cycle_ctx,
|
||||
run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
|
||||
&cycle_budget,
|
||||
run_data_scanner_cycle_with_budget(
|
||||
&cycle_ctx,
|
||||
&storeapi,
|
||||
&mut cycle_info,
|
||||
&mut cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget.clone(),
|
||||
),
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
.await
|
||||
.unwrap_or(ScannerCycleOutcome::Failed);
|
||||
{
|
||||
ScannerCycleWaitOutcome::Completed(outcome) => outcome,
|
||||
ScannerCycleWaitOutcome::LockLost => {
|
||||
record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await;
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Cancelled => {
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
cycle_info: &mut cycle_info,
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
},
|
||||
worker_stopped,
|
||||
&mut guard,
|
||||
)
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded);
|
||||
deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_)));
|
||||
dirty_usage_generation_seen = dirty_generation_before_cycle;
|
||||
@@ -1916,13 +2093,49 @@ async fn run_data_scanner_with_maintenance_state(
|
||||
}
|
||||
let dirty_generation_before_cycle = dirty_usage_generation();
|
||||
let cycle_ctx = ctx.child_token();
|
||||
let outcome = await_scanner_cycle_with_lock_fence(
|
||||
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
|
||||
let outcome = match await_scanner_cycle_with_budget_fence(
|
||||
&cycle_ctx,
|
||||
run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch),
|
||||
&cycle_budget,
|
||||
run_data_scanner_cycle_with_budget(
|
||||
&cycle_ctx,
|
||||
&storeapi,
|
||||
&mut cycle_info,
|
||||
&mut cycle_revision,
|
||||
leader_epoch,
|
||||
cycle_budget.clone(),
|
||||
),
|
||||
guard.lock_lost_notified(),
|
||||
)
|
||||
.await
|
||||
.unwrap_or(ScannerCycleOutcome::Failed);
|
||||
{
|
||||
ScannerCycleWaitOutcome::Completed(outcome) => outcome,
|
||||
ScannerCycleWaitOutcome::LockLost => {
|
||||
record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await;
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Cancelled => {
|
||||
global_metrics().set_cycle(None).await;
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
cycle_info: &mut cycle_info,
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
},
|
||||
worker_stopped,
|
||||
&mut guard,
|
||||
)
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded);
|
||||
deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_)));
|
||||
dirty_usage_generation_seen = dirty_generation_before_cycle;
|
||||
@@ -2219,7 +2432,12 @@ pub(crate) use activity::{
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
#[cfg(test)]
|
||||
pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
|
||||
pub(crate) use cycle_state::{current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence};
|
||||
pub use cycle_state::{
|
||||
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, reset_scanner_cycle_recovery, scanner_cycle_recovery_status,
|
||||
};
|
||||
pub(crate) use cycle_state::{
|
||||
current_scanner_leader_epoch, decode_persisted_scanner_cycle_fence, load_scanner_cycle_state_for_startup,
|
||||
};
|
||||
pub use heal_info::{BackgroundHealInfo, read_background_heal_info, save_background_heal_info};
|
||||
pub use usage_store::store_data_usage_in_backend;
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user