diff --git a/.config/e2e-full-selection.txt b/.config/e2e-full-selection.txt index bb9022327..dfad0f0bd 100644 --- a/.config/e2e-full-selection.txt +++ b/.config/e2e-full-selection.txt @@ -1,2 +1,2 @@ -sha256-darwin=b4ae71aa894e5c7795ae3eb8116f1777a7601d0f5db3898be2e48faf3329bd9b -sha256-linux=433debd9d9defa832986269abdf0f1d131597b2d7a417ce930e17c1fd47d85ba +sha256-darwin=9f767b37ed8b1c82da62ea441462d75487785c8086e56f08fb6f6cd89c6e2e52 +sha256-linux=fbdaf42b220958d4b1e8880e0f8b5a7992d38e21051bb60596dd4538424757d6 diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 1dec7db3e..626df9b84 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -36,6 +36,7 @@ script-tests: ## Run shell script tests ./scripts/test_manual_transition_runbooks.sh ./scripts/check_embedded_secrets.sh --self-test python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test python3 ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh python3 ./scripts/check_object_data_cache_follower_samples.py --self-test diff --git a/.github/actions/schedule-failure-issue/action.yml b/.github/actions/schedule-failure-issue/action.yml index 60e938690..d505387e4 100644 --- a/.github/actions/schedule-failure-issue/action.yml +++ b/.github/actions/schedule-failure-issue/action.yml @@ -14,9 +14,10 @@ name: "Schedule Failure Issue" description: >- - Open (or update) a tracking issue when a scheduled workflow run fails. + Open (or update) a tracking issue when a scheduled workflow run fails or + does not complete normally. Dedupes by workflow name: if an open issue titled - "[scheduled-failure] " already exists, the failure is + "[scheduled-failure] " already exists, the result is appended as a comment; otherwise a new issue is created. This is the single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8). @@ -38,6 +39,30 @@ inputs: Set to an empty string to skip labeling. required: false default: "infrastructure" + source-run-id: + description: "Run ID to report. Defaults to the current workflow run." + required: false + default: ${{ github.run_id }} + source-run-attempt: + description: "Run attempt to report. Defaults to the current attempt." + required: false + default: ${{ github.run_attempt }} + source-event: + description: "Trigger event of the run being reported." + required: false + default: ${{ github.event_name }} + source-ref-name: + description: "Ref name of the run being reported." + required: false + default: ${{ github.ref_name }} + source-sha: + description: "Commit SHA of the run being reported." + required: false + default: ${{ github.sha }} + details-file: + description: "Optional Markdown file appended to the issue body." + required: false + default: "" runs: using: "composite" @@ -48,17 +73,22 @@ runs: GH_TOKEN: ${{ inputs.github-token }} WORKFLOW_NAME: ${{ inputs.workflow-name }} ISSUE_LABEL: ${{ inputs.label }} + SOURCE_RUN_ID: ${{ inputs.source-run-id }} + SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }} + SOURCE_EVENT: ${{ inputs.source-event }} + SOURCE_REF_NAME: ${{ inputs.source-ref-name }} + SOURCE_SHA: ${{ inputs.source-sha }} + DETAILS_FILE: ${{ inputs.details-file }} run: | set -euo pipefail title="[scheduled-failure] ${WORKFLOW_NAME}" - run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}" - # Failed job names for this run attempt. The alert job runs while the - # run as a whole is still in progress, so inspect the jobs that have - # already completed with a non-success conclusion. + # Inspect the reported run attempt. It can be the current in-workflow + # failure or a completed run observed by the external watchdog. failed_jobs="$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \ --paginate \ --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled") @@ -67,15 +97,26 @@ runs: failed_jobs="- (failed job not recorded yet — see the run page)" fi + details="" + if [ -n "${DETAILS_FILE}" ]; then + if [ -f "${DETAILS_FILE}" ]; then + details="$(cat "${DETAILS_FILE}")" + else + details="Details file was not available: \`${DETAILS_FILE}\`" + fi + fi + body="$(cat <- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml index a9902fc4d..660d96b09 100644 --- a/.github/workflows/cache-warm.yml +++ b/.github/workflows/cache-warm.yml @@ -94,6 +94,9 @@ concurrency: env: CARGO_TERM_COLOR: always + # Swatinem/rust-cache hashes every RUST* variable. Keep this aligned with + # ci.yml or the writer and readers use disjoint cache keys. + RUST_BACKTRACE: 1 jobs: # Readers: test-and-lint, test-ilm-integration-serial, build-rustfs-debug-binary, @@ -101,7 +104,7 @@ jobs: warm-ci-dev: name: Warm ci-dev runs-on: sm-standard-4 - timeout-minutes: 90 + timeout-minutes: 120 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" steps: @@ -191,7 +194,7 @@ jobs: warm-ci-feat-rio: name: Warm ci-feat-rio runs-on: sm-standard-4 - timeout-minutes: 90 + timeout-minutes: 120 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" steps: @@ -219,7 +222,7 @@ jobs: warm-ci-feat-proto: name: Warm ci-feat-proto runs-on: sm-standard-4 - timeout-minutes: 90 + timeout-minutes: 120 env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" steps: diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index 85ce9b32f..de78e7f7f 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -126,7 +126,10 @@ jobs: run: ./scripts/check_embedded_secrets.sh - name: Check test wiring - run: python3 ./scripts/check_test_wiring.py + run: | + python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a06e4667..ae74ec40a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ on: merge_group: types: [ checks_requested ] schedule: - - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC + - cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC workflow_dispatch: permissions: @@ -161,7 +161,10 @@ jobs: run: ./scripts/check_embedded_secrets.sh - name: Check test wiring - run: python3 ./scripts/check_test_wiring.py + run: | + python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh @@ -1032,3 +1035,37 @@ jobs: path: artifacts/s3tests-single/** if-no-files-found: ignore retention-days: 3 + + alert-on-failure: + name: Alert on scheduled failure + needs: + - typos + - quick-checks + - test-and-lint + - test-ilm-integration-serial + - test-and-lint-rio-v2 + - test-and-lint-protocols + - build-rustfs-debug-binary + - build-rustfs-debug-binary-rio-v2 + - uring-integration + - e2e-tests + - e2e-full + - e2e-tests-rio-v2 + - s3-implemented-tests + - s3-lifecycle-behavior-tests + if: >- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index baa3c87a7..ece00c2eb 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,7 +37,7 @@ on: # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17), # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes. - - cron: "0 7 * * 0" + - cron: "43 7 * * 0" # Only alert-on-failure needs more than read access; it declares its own # job-level `issues: write`. diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml index 837d9f79f..145ad317e 100644 --- a/.github/workflows/e2e-replication-nightly.yml +++ b/.github/workflows/e2e-replication-nightly.yml @@ -40,7 +40,7 @@ on: schedule: # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00), # stale (01:30) and performance-ab (06:00). - - cron: "0 4 * * *" + - cron: "29 4 * * *" # Only alert-on-failure needs more than read access; it declares its own # job-level `issues: write`. @@ -196,6 +196,9 @@ jobs: cache-save-if: 'false' install-build-packaging-tools: 'false' + - name: Verify protocol socket oracle + run: ss -tn state CLOSE-WAIT >/dev/null + # The suite owns fixed protocol ports and serializes its internal cases. - name: Verify protocol e2e membership env: diff --git a/.github/workflows/e2e-s3tests.yml b/.github/workflows/e2e-s3tests.yml index 1e3ad90b6..1be61d73e 100644 --- a/.github/workflows/e2e-s3tests.yml +++ b/.github/workflows/e2e-s3tests.yml @@ -90,10 +90,14 @@ on: description: "Optional pytest -m expression" required: false default: "" + testexpr: + description: "Optional pytest -k expression" + required: false + default: "" schedule: # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the # single-node and the 4-node distributed topologies (matrix below). - - cron: "0 2 * * 0" + - cron: "19 2 * * 0" env: # main user @@ -116,6 +120,7 @@ env: XDIST: ${{ github.event.inputs.xdist || '4' }} MAXFAIL: ${{ github.event.inputs.maxfail || '0' }} MARKEXPR: ${{ github.event.inputs.markexpr || '' }} + TESTEXPR: ${{ github.event.inputs.testexpr || '' }} S3_SHARD_COUNT: ${{ github.event_name == 'schedule' && '4' || github.event.inputs.shard-count || '1' }} TEST_TIMEOUT: "300" @@ -269,14 +274,20 @@ jobs: EOF cat > haproxy.cfg <<'EOF' + global + log stdout format raw local0 info + defaults mode http + log global + log-format '%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %HM %HP' timeout connect 5s timeout client 30s timeout server 30s frontend fe_s3 bind *:9000 + option http-buffer-request default_backend be_s3 backend be_s3 @@ -314,6 +325,7 @@ jobs: XDIST="${XDIST}" \ MAXFAIL="${MAXFAIL}" \ MARKEXPR="${MARKEXPR}" \ + TESTEXPR="${TESTEXPR}" \ ./scripts/s3-tests/run.sh - name: Publish compatibility report diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index d41a1107f..6aa780fc6 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -30,7 +30,7 @@ on: - "Cargo.lock" - ".github/workflows/fuzz.yml" schedule: - - cron: "0 2 * * *" + - cron: "17 2 * * *" workflow_dispatch: inputs: profile: diff --git a/.github/workflows/minio-interop.yml b/.github/workflows/minio-interop.yml index 3ee33e9bf..5f105ac41 100644 --- a/.github/workflows/minio-interop.yml +++ b/.github/workflows/minio-interop.yml @@ -121,3 +121,21 @@ jobs: cargo nextest run --run-ignored ignored-only --no-tests=fail \ -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \ -E "$INTEROP_FILTER" + + alert-on-failure: + name: Alert on scheduled failure + needs: [minio-interop] + if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/mint.yml b/.github/workflows/mint.yml index dd9acc6f8..25ae02916 100644 --- a/.github/workflows/mint.yml +++ b/.github/workflows/mint.yml @@ -45,13 +45,6 @@ # docker-capable self-hosted `dind-sm-standard-2` label was the alternative but # has fewer cores and reintroduces fleet-state risk for no reliability gain. -# DISABLED. This workflow is switched off in the repository's Actions settings -# (state: disabled_manually) and does not run on any trigger, including its cron -# and workflow_dispatch. That state lives in GitHub's UI and is invisible when -# reading this file, which has already misled at least one audit — hence this -# banner. Re-enabling is a UI action; anyone doing so should first check that the -# workflow still matches the current CI layout. See rustfs/backlog#1603. -# name: mint on: @@ -70,13 +63,13 @@ on: - core - full mint-image: - description: "Mint image reference" + description: "Mint image reference (empty = pinned default)" required: false - default: "minio/mint:edge" + default: "" schedule: # Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to # 3h) has finished, so the two never contend for the same runner pool. - - cron: "0 6 * * 0" + - cron: "41 6 * * 0" env: S3_ACCESS_KEY: rustfsadmin-ci diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml index 1f7c2d488..946761e54 100644 --- a/.github/workflows/nightly-gnu.yml +++ b/.github/workflows/nightly-gnu.yml @@ -16,7 +16,7 @@ name: Nightly GNU Build on: schedule: - - cron: "0 0 * * *" + - cron: "7 0 * * *" timezone: "Asia/Shanghai" workflow_dispatch: @@ -194,3 +194,23 @@ jobs: - name: Run HA leader failover live checks (three-node Raft cluster in Docker) run: bash scripts/test/vault_ha_kms_live.sh + + alert-on-failure: + name: Alert on scheduled failure + needs: [build, kms-vault-lane, kms-vault-ha-failover] + if: >- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml index ac8e8d7c5..de4986388 100644 --- a/.github/workflows/performance-ab.yml +++ b/.github/workflows/performance-ab.yml @@ -22,22 +22,15 @@ # correctness cost (e.g. the #4221 fsync durability fix) is recorded, not # blocked (rustfs/backlog#935 correction 1). -# DISABLED. This workflow is switched off in the repository's Actions settings -# (state: disabled_manually) and does not run on any trigger, including its cron -# and workflow_dispatch. That state lives in GitHub's UI and is invisible when -# reading this file, which has already misled at least one audit — hence this -# banner. Re-enabling is a UI action; anyone doing so should first check that the -# workflow still matches the current CI layout. See rustfs/backlog#1603. -# name: Performance A/B on: schedule: - - cron: "0 6 * * *" # 06:00 UTC nightly, against main + - cron: "31 6 * * *" # 06:31 UTC nightly, against main workflow_dispatch: inputs: duration: - description: "warp duration per round (short by default to fit the double-build budget)" + description: "warp duration per round" required: false default: "12s" type: string @@ -46,12 +39,8 @@ on: required: false default: false type: boolean - push: - # Every main commit pre-builds and caches its release binary (perf-3) so the - # nightly A/B restores a ready baseline instead of paying the double build. - branches: [main] - permissions: + actions: read contents: read env: @@ -59,83 +48,19 @@ env: RUST_BACKTRACE: 1 jobs: - # perf-3: on every push to main, build the release binary once and cache it - # keyed by commit SHA (rustfs-baseline-). The warp-ab measurements - # restore this instead of paying the ~32min-per-side source - # build. That double build is what pushed the expanded 24-cell nightly past its - # ceiling — 2026-07-11..07-14 all cancelled on the 120min timeout. Incremental - # builds off the shared cargo cache keep each push cheap, and building on the - # same sm-standard-2 runner the A/B measures on guarantees the cached binary is - # ABI-identical. Do NOT source this from build.yml's per-merge artifact: those - # are cancelled ~7/8 of the time and are not a reliable baseline. - build-baseline-cache: - name: Build + cache baseline binary - if: github.event_name == 'push' - runs-on: sm-standard-2 - # Latest-wins: consumers only ever restore the binary for the *current* - # origin/main tip, so when pushes land faster than the ~65min build, a - # superseded build's output is dead weight — cancel it instead of stacking - # hour-long jobs on the shared runner pool. A skipped intermediate SHA at - # most costs one same-commit self-heal in the A/B job. - concurrency: - group: perf-baseline-build-main - cancel-in-progress: true - # #4806 put thin LTO + codegen-units=1 on [profile.release], pushing a - # single release build past 60min on this runner — every cache build on - # 2026-07-15 died on the old 60min ceiling ("exceeded the maximum execution - # time of 1h0m0s") and the cache never populated. The measured binary must - # keep the production profile, so the budget absorbs the build instead. - timeout-minutes: 100 - steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - - name: Setup Rust environment - uses: ./.github/actions/setup - with: - rust-version: stable - cache-shared-key: warp-ab-${{ hashFiles('**/Cargo.lock') }} - cache-save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Build release rustfs - run: cargo build --release --bin rustfs - - - name: Stage binary for cache - run: | - set -euo pipefail - mkdir -p baseline-bin - cp target/release/rustfs baseline-bin/rustfs - - - name: Cache baseline binary by SHA - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: baseline-bin/rustfs - key: rustfs-baseline-${{ github.sha }} - warp-ab: name: Warp A/B budget gate - # Always run on schedule / manual dispatch. Never on push — that event only - # feeds build-baseline-cache above. - if: >- - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' runs-on: sm-standard-2 - # With perf-3's cached baseline binary the common (cache-hit) nightly is - # measurement-only and finishes well under 50min. This ceiling stays - # generous only to absorb the same-commit cache-miss self-heal (~65min - # single build with the post-#4806 LTO profile + measurement). A timeout - # surfaces via the alert-on-failure job (it fires on cancelled/timed-out, - # not just failure). perf-6 recalibrates the budget once the noise study - # lands. - timeout-minutes: 120 + # A normal nightly restores the last successful binary and builds only the + # candidate; daily access keeps that cache warm. A cache miss may build both + # and needs room for the A/B run plus artifact and cache publication. + timeout-minutes: 180 steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false - fetch-depth: 0 # baseline is built from origin/main + fetch-depth: 0 # baseline may be an earlier successful scheduled head - name: Setup Rust environment uses: ./.github/actions/setup @@ -163,24 +88,55 @@ jobs: fi echo "allow_regression=$allow" >> "$GITHUB_OUTPUT" - # perf-3: resolve the commits so the cache can be keyed by SHA. The - # baseline is origin/main; the candidate is the checked-out ref. On the - # nightly (checkout == main) they are the same commit, so one cached binary - # serves both phases and the run does zero source builds. + # A failed regression run must keep comparing against the last known-good + # scheduled head. Otherwise the next nightly would absorb the regression + # into its baseline and turn green without a fix. + - name: Find last successful scheduled baseline + id: scheduled_baseline + if: github.event_name == 'schedule' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + result-encoding: string + script: | + const { data } = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "performance-ab.yml", + event: "schedule", + status: "success", + per_page: 1, + }); + return data.workflow_runs[0]?.head_sha ?? ""; + + # Manual runs compare a selected ref with current main. Scheduled runs + # compare current main with the last successful scheduled head. With no + # history, the first run measures the candidate against itself and seeds + # that head only if the complete rig succeeds. - name: Resolve baseline / candidate commits id: commits + env: + SCHEDULED_BASELINE_SHA: ${{ steps.scheduled_baseline.outputs.result }} run: | set -euo pipefail - baseline_sha="$(git rev-parse origin/main)" candidate_sha="$(git rev-parse HEAD)" + if [[ "${{ github.event_name }}" == "schedule" ]]; then + baseline_sha="${SCHEDULED_BASELINE_SHA:-$candidate_sha}" + if ! git merge-base --is-ancestor "$baseline_sha" "$candidate_sha"; then + echo "::error::scheduled baseline $baseline_sha is not an ancestor of candidate $candidate_sha" >&2 + exit 1 + fi + else + baseline_sha="$(git rev-parse origin/main)" + fi + git cat-file -e "${baseline_sha}^{commit}" echo "baseline_sha=$baseline_sha" >> "$GITHUB_OUTPUT" echo "candidate_sha=$candidate_sha" >> "$GITHUB_OUTPUT" echo "baseline commit: $baseline_sha" echo "candidate commit: $candidate_sha" - # Exact-key restore of the baseline binary built by build-baseline-cache - # when origin/main last landed. A miss (binary evicted or not built yet) - # leaves cache-hit unset and the rig falls back to a source build. + # Exact-key restore of the candidate binary saved by its successful + # scheduled run. A miss leaves cache-hit unset and falls back to a source + # build of that known-good head. - name: Restore cached baseline binary id: baseline_cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 @@ -270,11 +226,11 @@ jobs: elif [[ "$selfheal_built" == "true" ]]; then base_src="source build (cache self-heal, saved as rustfs-baseline-$baseline_sha)" else - base_src="isolated origin/main source build (saved as rustfs-baseline-$baseline_sha)" + base_src="isolated baseline source build (saved as rustfs-baseline-$baseline_sha)" fi if [[ "$candidate_sha" == "$baseline_sha" ]]; then - # Nightly on main: the candidate is the same commit as the baseline, - # so reuse the one binary for both phases and skip all builds. + # No commits landed since the last successful baseline, so reuse + # the one binary for both phases and measure only rig drift. args+=(--candidate-bin "$base_bin") cand_src="same binary as baseline (same commit)" elif [[ "$candidate_built" == "true" ]]; then @@ -362,6 +318,23 @@ jobs: fi } >> "$GITHUB_STEP_SUMMARY" + - name: Stage successful candidate baseline + if: >- + steps.ab.outputs.status == '0' && + steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha + run: | + set -euo pipefail + cp candidate-bin/rustfs baseline-bin/rustfs + + - name: Cache successful candidate baseline + if: >- + steps.ab.outputs.status == '0' && + steps.commits.outputs.baseline_sha != steps.commits.outputs.candidate_sha + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: baseline-bin/rustfs + key: rustfs-baseline-${{ steps.commits.outputs.candidate_sha }} + # Scheduled failure alerting is handled by the alert-on-failure job below # (perf-2 consuming ci-8's schedule-failure-issue composite action). diff --git a/.github/workflows/runner-hygiene.yml b/.github/workflows/runner-hygiene.yml index c231b5706..bdebf2c77 100644 --- a/.github/workflows/runner-hygiene.yml +++ b/.github/workflows/runner-hygiene.yml @@ -30,7 +30,7 @@ name: Runner Hygiene on: schedule: - - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron) + - cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC workflow_dispatch: permissions: diff --git a/.github/workflows/scheduled-validation-freshness.yml b/.github/workflows/scheduled-validation-freshness.yml new file mode 100644 index 000000000..eef340869 --- /dev/null +++ b/.github/workflows/scheduled-validation-freshness.yml @@ -0,0 +1,57 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Scheduled Validation Freshness + +on: + schedule: + - cron: "47 23 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: scheduled-validation-freshness + cancel-in-progress: false + +jobs: + check-freshness: + name: Check scheduled validation freshness + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Check latest scheduled runs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + python3 scripts/check_scheduled_validation_freshness.py \ + --report "${RUNNER_TEMP}/scheduled-validation-freshness.md" + status=$? + cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}" + exit "${status}" + - name: Open or update freshness issue + if: failure() + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + details-file: ${{ runner.temp }}/scheduled-validation-freshness.md diff --git a/.github/workflows/scheduled-validation-watchdog.yml b/.github/workflows/scheduled-validation-watchdog.yml new file mode 100644 index 000000000..e778ec640 --- /dev/null +++ b/.github/workflows/scheduled-validation-watchdog.yml @@ -0,0 +1,63 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Scheduled Validation Watchdog + +on: + workflow_run: + workflows: + - "Security Audit" + - "Build and Release" + - "Continuous Integration" + - "coverage" + - "e2e-nightly" + - "e2e-s3tests" + - "Fuzz" + - "mint" + - "minio-interop" + - "Nightly GNU Build" + - "Performance A/B" + - "Runner Hygiene" + types: [completed] + +permissions: + contents: read + +jobs: + alert-on-incomplete-run: + name: Alert on incomplete scheduled run + if: >- + github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.conclusion != 'success' && + github.event.workflow_run.conclusion != 'failure' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update incomplete-run issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + workflow-name: ${{ github.event.workflow_run.name }} + source-run-id: ${{ github.event.workflow_run.id }} + source-run-attempt: ${{ github.event.workflow_run.run_attempt }} + source-event: ${{ github.event.workflow_run.event }} + source-ref-name: ${{ github.event.workflow_run.head_branch }} + source-sha: ${{ github.event.workflow_run.head_sha }} diff --git a/crates/common/src/metrics.rs b/crates/common/src/metrics.rs index 51eef967a..813e39ac3 100644 --- a/crates/common/src/metrics.rs +++ b/crates/common/src/metrics.rs @@ -901,6 +901,10 @@ pub struct Metrics { scanner_cycle_max_duration_millis: AtomicU64, scanner_cycle_max_objects: AtomicU64, scanner_cycle_max_directories: AtomicU64, + scanner_cycle_timeout_total: AtomicU64, + scanner_cycle_recovery_required_total: AtomicU64, + scanner_cycle_last_progress_age_seconds: AtomicU64, + scanner_leader_lease_without_progress: AtomicBool, scanner_bitrot_cycle_enabled: AtomicBool, scanner_bitrot_cycle_millis: AtomicU64, scanner_checkpoint: Mutex>, @@ -1370,6 +1374,14 @@ pub struct ScannerMetricsReport { #[serde(default)] pub cycle_max_directories: u64, #[serde(default)] + pub cycle_timeout_total: u64, + #[serde(default)] + pub cycle_recovery_required_total: u64, + #[serde(default)] + pub cycle_last_progress_age: u64, + #[serde(default)] + pub leader_lease_without_progress: bool, + #[serde(default)] pub bitrot_cycle_enabled: bool, #[serde(default)] pub bitrot_cycle_seconds: f64, @@ -1430,6 +1442,9 @@ const OTEL_SCANNER_BUCKETS_SCANNED: &str = "rustfs_scanner_buckets_scanned_total const OTEL_SCANNER_CYCLES: &str = "rustfs_scanner_cycles_total"; const OTEL_SCANNER_CYCLE_DURATION_SECONDS: &str = "rustfs_scanner_cycle_duration_seconds"; const OTEL_SCANNER_BUCKET_DRIVE_DURATION_SECONDS: &str = "rustfs_scanner_bucket_drive_duration_seconds"; +const OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL: &str = "rustfs_scanner_cycle_timeout_total"; +const OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE: &str = "rustfs_scanner_cycle_last_progress_age"; +const OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS: &str = "rustfs_scanner_leader_lease_without_progress"; fn scan_cycle_result_label(result: u8) -> &'static str { match result { @@ -1913,6 +1928,10 @@ impl Metrics { scanner_cycle_max_duration_millis: AtomicU64::new(0), scanner_cycle_max_objects: AtomicU64::new(0), scanner_cycle_max_directories: AtomicU64::new(0), + scanner_cycle_timeout_total: AtomicU64::new(0), + scanner_cycle_recovery_required_total: AtomicU64::new(0), + scanner_cycle_last_progress_age_seconds: AtomicU64::new(0), + scanner_leader_lease_without_progress: AtomicBool::new(false), scanner_bitrot_cycle_enabled: AtomicBool::new(false), scanner_bitrot_cycle_millis: AtomicU64::new(0), scanner_checkpoint: Mutex::new(None), @@ -2412,12 +2431,29 @@ impl Metrics { .store(cycle_max_objects.unwrap_or_default(), Ordering::Relaxed); self.scanner_cycle_max_directories .store(cycle_max_directories.unwrap_or_default(), Ordering::Relaxed); + self.scanner_leader_lease_without_progress.store(false, Ordering::Relaxed); + self.scanner_cycle_last_progress_age_seconds.store(0, Ordering::Relaxed); + metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(0.0); + metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(0.0); self.scanner_bitrot_cycle_enabled .store(bitrot_cycle.is_some(), Ordering::Relaxed); self.scanner_bitrot_cycle_millis .store(bitrot_cycle.map(duration_millis_saturated).unwrap_or_default(), Ordering::Relaxed); } + pub fn record_scanner_cycle_timeout(&self, recovery_required: bool, progress_age: Duration) { + self.scanner_cycle_timeout_total.fetch_add(1, Ordering::Relaxed); + if recovery_required { + self.scanner_cycle_recovery_required_total.fetch_add(1, Ordering::Relaxed); + } + self.scanner_cycle_last_progress_age_seconds + .store(progress_age.as_secs(), Ordering::Relaxed); + self.scanner_leader_lease_without_progress.store(true, Ordering::Relaxed); + metrics::counter!(OTEL_SCANNER_CYCLE_TIMEOUT_TOTAL).increment(1); + metrics::gauge!(OTEL_SCANNER_CYCLE_LAST_PROGRESS_AGE).set(progress_age.as_secs_f64()); + metrics::gauge!(OTEL_SCANNER_LEADER_LEASE_WITHOUT_PROGRESS).set(1.0); + } + pub fn record_scanner_set_scan_state(&self, concurrency_limit: Option, queued: Option, active: Option) { if let Some(concurrency_limit) = concurrency_limit { self.scanner_set_scan_concurrency_limit @@ -3265,6 +3301,10 @@ impl Metrics { m.cycle_max_duration_seconds = self.scanner_cycle_max_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0; m.cycle_max_objects = self.scanner_cycle_max_objects.load(Ordering::Relaxed); m.cycle_max_directories = self.scanner_cycle_max_directories.load(Ordering::Relaxed); + m.cycle_timeout_total = self.scanner_cycle_timeout_total.load(Ordering::Relaxed); + m.cycle_recovery_required_total = self.scanner_cycle_recovery_required_total.load(Ordering::Relaxed); + m.cycle_last_progress_age = self.scanner_cycle_last_progress_age_seconds.load(Ordering::Relaxed); + m.leader_lease_without_progress = self.scanner_leader_lease_without_progress.load(Ordering::Relaxed); m.bitrot_cycle_enabled = self.scanner_bitrot_cycle_enabled.load(Ordering::Relaxed); m.bitrot_cycle_seconds = self.scanner_bitrot_cycle_millis.load(Ordering::Relaxed) as f64 / 1000.0; m.scan_checkpoint = match self.scanner_checkpoint.lock() { @@ -4926,4 +4966,20 @@ mod tests { assert!(!report.bitrot_cycle_enabled); assert_eq!(report.bitrot_cycle_seconds, 0.0); } + + #[tokio::test] + async fn scanner_cycle_timeout_metrics_reset_for_a_new_cycle() { + let metrics = Metrics::new(); + metrics.record_scanner_cycle_timeout(true, Duration::from_secs(17)); + let timed_out = metrics.report().await; + assert_eq!(timed_out.cycle_timeout_total, 1); + assert_eq!(timed_out.cycle_last_progress_age, 17); + assert!(timed_out.leader_lease_without_progress); + + metrics.record_scanner_cycle_config(Duration::from_secs(60), None, Some(Duration::from_secs(1)), None, None); + let current = metrics.report().await; + assert_eq!(current.cycle_timeout_total, 1); + assert_eq!(current.cycle_last_progress_age, 0); + assert!(!current.leader_lease_without_progress); + } } diff --git a/crates/config/README.md b/crates/config/README.md index 02cf81d52..338b85dbf 100644 --- a/crates/config/README.md +++ b/crates/config/README.md @@ -84,6 +84,12 @@ Current guidance: - `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical) - `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical) +Scanner cycle budget controls: + +- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance. +- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`. +- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted. + ## Mmap read environment aliases - `RUSTFS_OBJECT_MMAP_READ_ENABLE` (canonical) diff --git a/crates/config/src/constants/scanner.rs b/crates/config/src/constants/scanner.rs index 8086c3229..953ef789b 100644 --- a/crates/config/src/constants/scanner.rs +++ b/crates/config/src/constants/scanner.rs @@ -143,9 +143,12 @@ pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS"; /// Default scanner speed preset. pub const DEFAULT_SCANNER_SPEED: &str = "default"; -/// Default scanner cycle runtime budget. -/// `0` keeps the existing unbounded per-cycle behavior. -pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 0; +/// Default scanner cycle runtime budget when no override is configured. +/// +/// An explicit `0` remains the compatibility escape hatch for an unbounded +/// cycle. Keeping the unset default finite prevents a stalled scanner I/O +/// operation from holding the leader lease forever. +pub const DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS: u64 = 30 * 60; /// Default scanner per-cycle object budget. /// `0` keeps the existing unbounded per-cycle behavior. diff --git a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs index fecba2b89..c0f0f0181 100644 --- a/crates/e2e_test/src/kms/bucket_default_encryption_test.rs +++ b/crates/e2e_test/src/kms/bucket_default_encryption_test.rs @@ -37,7 +37,7 @@ async fn test_bucket_default_sse_s3_put_object() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { - let total_deadline = Duration::from_secs(5); + wait_for_kms_ready_with_timeout(base_url, access_key, secret_key, Duration::from_secs(5)).await +} + +async fn wait_for_kms_ready_with_timeout( + base_url: &str, + access_key: &str, + secret_key: &str, + total_deadline: Duration, +) -> Result<(), Box> { let start = tokio::time::Instant::now(); + let deadline = start + total_deadline; let mut backoff = Duration::from_millis(200); let max_backoff = Duration::from_secs(1); - let mut first_attempt = true; loop { - if !first_attempt { - if start.elapsed() >= total_deadline { - return Err("KMS failed to become ready within 5 seconds".into()); - } - sleep(backoff).await; - backoff = (backoff * 2).min(max_backoff); - } - first_attempt = false; - - match get_kms_status(base_url, access_key, secret_key).await { - Ok(status) => { - info!("KMS is ready (status: {})", status); - return Ok(()); - } - Err(e) => { - if start.elapsed() >= total_deadline { - return Err(format!("KMS did not become ready within 5 s: last error: {e}").into()); + match tokio::time::timeout_at(deadline, get_kms_status(base_url, access_key, secret_key)).await { + Ok(Ok(status)) => { + let backend_status = serde_json::from_str::(&status) + .ok() + .and_then(|value| value.get("backend_status")?.as_str().map(str::to_owned)); + if backend_status.as_deref() == Some("healthy") { + info!("KMS is ready (status: {})", status); + return Ok(()); } - warn!(error = %e, elapsed_ms = start.elapsed().as_millis() as u64, "KMS not ready yet, retrying…"); + warn!( + backend_status = backend_status.as_deref().unwrap_or("missing"), + elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX), + "KMS not ready yet, retrying…" + ); } + Ok(Err(e)) => { + let elapsed_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX); + warn!(error = %e, elapsed_ms, "KMS not ready yet, retrying…"); + } + Err(_) => return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()), } + + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(format!("KMS failed to become ready within {} ms", total_deadline.as_millis()).into()); + } + sleep((now + backoff).min(deadline) - now).await; + backoff = (backoff * 2).min(max_backoff); + } +} + +#[cfg(test)] +mod readiness_tests { + use super::{wait_for_kms_ready, wait_for_kms_ready_with_timeout}; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + #[tokio::test] + async fn kms_readiness_retries_http_success_until_backend_is_healthy() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server"); + let address = listener.local_addr().expect("read readiness test server address"); + let requests = Arc::new(AtomicUsize::new(0)); + let server_requests = Arc::clone(&requests); + let server = tokio::spawn(async move { + for backend_status in ["error", "healthy"] { + let (mut socket, _) = listener.accept().await.expect("accept readiness request"); + let mut request = Vec::new(); + let mut chunk = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket.read(&mut chunk).await.expect("read readiness request"); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + } + server_requests.fetch_add(1, Ordering::SeqCst); + + let body = format!(r#"{{"backend_status":"{backend_status}"}}"#); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + socket.write_all(response.as_bytes()).await.expect("write readiness response"); + } + }); + + wait_for_kms_ready(&format!("http://{address}"), "access-key", "secret-key") + .await + .expect("KMS should become ready after the healthy response"); + + let observed_requests = requests.load(Ordering::SeqCst); + server.abort(); + assert_eq!(observed_requests, 2, "an HTTP 200 unhealthy status must be retried"); + } + + #[tokio::test] + async fn kms_readiness_deadline_covers_a_stalled_status_request() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind readiness test server"); + let address = listener.local_addr().expect("read readiness test server address"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept readiness request"); + let mut request = [0_u8; 1024]; + let _ = socket.read(&mut request).await.expect("read readiness request"); + std::future::pending::<()>().await; + }); + + let result = tokio::time::timeout( + Duration::from_secs(1), + wait_for_kms_ready_with_timeout(&format!("http://{address}"), "access-key", "secret-key", Duration::from_millis(50)), + ) + .await + .expect("readiness helper must enforce its own deadline"); + + server.abort(); + assert!(result.is_err(), "a stalled status request must not outlive the readiness deadline"); } } diff --git a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs index 1cf19a565..85a013ab3 100644 --- a/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_self_copy_sse_test.rs @@ -61,7 +61,7 @@ async fn test_metadata_replace_self_copy_of_sse_object_stays_decryptable() { ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await.expect("KMS ready"); let client = kms_env.base_env.create_s3_client(); // Deliberately an UNVERSIONED bucket: that is the branch where the store layer can service @@ -160,7 +160,7 @@ async fn test_metadata_replace_self_copy_dropping_sse_rewrites_plaintext() { ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await.expect("KMS ready"); let client = kms_env.base_env.create_s3_client(); // Unversioned, and deliberately WITHOUT a bucket default-encryption rule, so the copy below @@ -256,7 +256,7 @@ async fn test_metadata_replace_self_copy_under_bucket_default_sse_stays_decrypta ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await.expect("KMS ready"); let client = kms_env.base_env.create_s3_client(); let bucket = "copy-object-self-copy-bucket-default-sse-test"; diff --git a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs index 3241a217d..c7a572e93 100644 --- a/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs +++ b/crates/e2e_test/src/kms/copy_object_version_restore_sse_test.rs @@ -56,7 +56,7 @@ async fn test_self_copy_of_historical_sse_s3_version_is_readable() { ) .await .expect("failed to start RustFS with local KMS"); - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await.expect("KMS ready"); let client = kms_env.base_env.create_s3_client(); let bucket = "copy-object-version-restore-sse-test"; diff --git a/crates/e2e_test/src/kms/encryption_metadata_test.rs b/crates/e2e_test/src/kms/encryption_metadata_test.rs index a316668f6..59501430b 100644 --- a/crates/e2e_test/src/kms/encryption_metadata_test.rs +++ b/crates/e2e_test/src/kms/encryption_metadata_test.rs @@ -87,7 +87,7 @@ async fn test_head_reports_managed_metadata_for_sse_s3() -> Result<(), Box Result<(), let mut kms_env = LocalKMSTestEnvironment::new().await?; let default_key_id = kms_env.start_rustfs_for_local_kms().await?; - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let s3_client = kms_env.base_env.create_s3_client(); kms_env.base_env.create_test_bucket(TEST_BUCKET).await?; @@ -250,7 +250,7 @@ async fn test_multipart_upload_writes_encrypted_data() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Bo let mut kms_env = LocalKMSTestEnvironment::new().await?; let _default_key_id = kms_env.start_rustfs_for_local_kms().await?; - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + kms_env.wait_for_kms_ready().await?; let s3_client = kms_env.base_env.create_s3_client(); kms_env.base_env.create_test_bucket(TEST_BUCKET).await?; @@ -187,7 +187,7 @@ async fn test_step3_multipart_upload_with_sse_s3() -> Result<(), Box Result<(), Box Result<(), Box &'static str { - match self { - TestCategory::CoreFunctionality => "core-functionality", - TestCategory::MultipartEncryption => "multipart-encryption", - TestCategory::EdgeCases => "edge-cases", - TestCategory::FaultRecovery => "fault-recovery", - TestCategory::Comprehensive => "comprehensive", - TestCategory::Performance => "performance", - } - } -} - -/// Test definition with metadata -#[derive(Debug, Clone)] -pub struct TestDefinition { - pub name: String, - pub description: String, - pub category: TestCategory, - pub estimated_duration: Duration, - pub is_critical: bool, -} - -impl TestDefinition { - pub fn new( - name: impl Into, - description: impl Into, - category: TestCategory, - estimated_duration: Duration, - is_critical: bool, - ) -> Self { - Self { - name: name.into(), - description: description.into(), - category, - estimated_duration, - is_critical, - } - } -} - -/// Test execution result -#[derive(Debug, Clone)] -pub struct TestResult { - pub test_name: String, - pub category: TestCategory, - pub success: bool, - pub duration: Duration, - pub error_message: Option, -} - -impl TestResult { - pub fn success(test_name: String, category: TestCategory, duration: Duration) -> Self { - Self { - test_name, - category, - success: true, - duration, - error_message: None, - } - } - - pub fn failure(test_name: String, category: TestCategory, duration: Duration, error: String) -> Self { - Self { - test_name, - category, - success: false, - duration, - error_message: Some(error), - } - } -} - -/// Comprehensive test suite configuration -#[derive(Debug, Clone)] -pub struct TestSuiteConfig { - pub categories: Vec, - pub include_critical_only: bool, - pub max_duration: Option, - pub parallel_execution: bool, -} - -impl Default for TestSuiteConfig { - fn default() -> Self { - Self { - categories: vec![ - TestCategory::CoreFunctionality, - TestCategory::MultipartEncryption, - TestCategory::EdgeCases, - TestCategory::FaultRecovery, - TestCategory::Comprehensive, - ], - include_critical_only: false, - max_duration: None, - parallel_execution: false, - } - } -} - -/// Unified KMS test suite runner -pub struct KMSTestSuite { - tests: Vec, - config: TestSuiteConfig, -} - -impl KMSTestSuite { - /// Create a new test suite with default configuration - pub fn new() -> Self { - let tests = vec![ - // Core Functionality Tests - TestDefinition::new( - "test_local_kms_end_to_end", - "End-to-end KMS test with all encryption types", - TestCategory::CoreFunctionality, - Duration::from_secs(60), - true, - ), - TestDefinition::new( - "test_local_kms_key_isolation", - "Test KMS key isolation and security", - TestCategory::CoreFunctionality, - Duration::from_secs(45), - true, - ), - // Multipart Encryption Tests - TestDefinition::new( - "test_local_kms_multipart_upload", - "Test large file multipart upload with encryption", - TestCategory::MultipartEncryption, - Duration::from_secs(120), - true, - ), - TestDefinition::new( - "test_step1_basic_single_file_encryption", - "Basic single file encryption test", - TestCategory::MultipartEncryption, - Duration::from_secs(30), - false, - ), - TestDefinition::new( - "test_step2_basic_multipart_upload_without_encryption", - "Basic multipart upload without encryption", - TestCategory::MultipartEncryption, - Duration::from_secs(45), - false, - ), - TestDefinition::new( - "test_step3_multipart_upload_with_sse_s3", - "Multipart upload with SSE-S3 encryption", - TestCategory::MultipartEncryption, - Duration::from_secs(60), - true, - ), - TestDefinition::new( - "test_step4_large_multipart_upload_with_encryption", - "Large file multipart upload with encryption", - TestCategory::MultipartEncryption, - Duration::from_secs(90), - false, - ), - TestDefinition::new( - "test_step5_all_encryption_types_multipart", - "All encryption types multipart test", - TestCategory::MultipartEncryption, - Duration::from_secs(120), - true, - ), - // Edge Cases Tests - TestDefinition::new( - "test_kms_zero_byte_file_encryption", - "Test encryption of zero-byte files", - TestCategory::EdgeCases, - Duration::from_secs(20), - false, - ), - TestDefinition::new( - "test_kms_single_byte_file_encryption", - "Test encryption of single-byte files", - TestCategory::EdgeCases, - Duration::from_secs(20), - false, - ), - TestDefinition::new( - "test_kms_multipart_boundary_conditions", - "Test multipart upload boundary conditions", - TestCategory::EdgeCases, - Duration::from_secs(45), - false, - ), - TestDefinition::new( - "test_kms_invalid_key_scenarios", - "Test invalid key scenarios", - TestCategory::EdgeCases, - Duration::from_secs(30), - false, - ), - TestDefinition::new( - "test_kms_concurrent_encryption", - "Test concurrent encryption operations", - TestCategory::EdgeCases, - Duration::from_secs(60), - false, - ), - TestDefinition::new( - "test_kms_key_validation_security", - "Test key validation security", - TestCategory::EdgeCases, - Duration::from_secs(30), - false, - ), - // Fault Recovery Tests - TestDefinition::new( - "test_kms_key_directory_unavailable", - "Test KMS when key directory is unavailable", - TestCategory::FaultRecovery, - Duration::from_secs(45), - false, - ), - TestDefinition::new( - "test_kms_corrupted_key_files", - "Test KMS with corrupted key files", - TestCategory::FaultRecovery, - Duration::from_secs(30), - false, - ), - TestDefinition::new( - "test_kms_multipart_upload_interruption", - "Test multipart upload interruption recovery", - TestCategory::FaultRecovery, - Duration::from_secs(60), - false, - ), - TestDefinition::new( - "test_kms_resource_constraints", - "Test KMS under resource constraints", - TestCategory::FaultRecovery, - Duration::from_secs(90), - false, - ), - // Comprehensive Tests - TestDefinition::new( - "test_comprehensive_kms_full_workflow", - "Full KMS workflow comprehensive test", - TestCategory::Comprehensive, - Duration::from_secs(300), - true, - ), - TestDefinition::new( - "test_comprehensive_stress_test", - "KMS stress test with large datasets", - TestCategory::Comprehensive, - Duration::from_secs(400), - false, - ), - TestDefinition::new( - "test_comprehensive_key_isolation", - "Comprehensive key isolation test", - TestCategory::Comprehensive, - Duration::from_secs(180), - false, - ), - TestDefinition::new( - "test_comprehensive_concurrent_operations", - "Comprehensive concurrent operations test", - TestCategory::Comprehensive, - Duration::from_secs(240), - false, - ), - TestDefinition::new( - "test_comprehensive_performance_benchmark", - "KMS performance benchmark test", - TestCategory::Comprehensive, - Duration::from_secs(360), - false, - ), - ]; - - Self { - tests, - config: TestSuiteConfig::default(), - } - } - - /// Configure the test suite - pub fn with_config(mut self, config: TestSuiteConfig) -> Self { - self.config = config; - self - } - - /// Filter tests based on category - pub fn filter_by_category(&self, category: &TestCategory) -> Vec<&TestDefinition> { - self.tests.iter().filter(|test| &test.category == category).collect() - } - - /// Filter tests based on criticality - pub fn filter_critical_tests(&self) -> Vec<&TestDefinition> { - self.tests.iter().filter(|test| test.is_critical).collect() - } - - /// Get test summary by category - pub fn get_category_summary(&self) -> std::collections::HashMap> { - let mut summary = std::collections::HashMap::new(); - for test in &self.tests { - summary.entry(test.category.clone()).or_insert_with(Vec::new).push(test); - } - summary - } - - /// Run the complete test suite - pub async fn run_test_suite(&self) -> Vec { - init_logging(); - info!("🚀 Starting unified KMS test suite"); - - let start_time = Instant::now(); - let mut results = Vec::new(); - - // Filter tests based on configuration - let tests_to_run: Vec<&TestDefinition> = self - .tests - .iter() - .filter(|test| self.config.categories.contains(&test.category)) - .filter(|test| !self.config.include_critical_only || test.is_critical) - .collect(); - - info!("📊 Test plan: {} test(s) scheduled", tests_to_run.len()); - for (i, test) in tests_to_run.iter().enumerate() { - info!(" {}. {} ({})", i + 1, test.name, test.category.as_str()); - } - - // Execute tests - for (i, test_def) in tests_to_run.iter().enumerate() { - info!("🧪 Running test {}/{}: {}", i + 1, tests_to_run.len(), test_def.name); - info!(" 📝 Description: {}", test_def.description); - info!(" 🏷️ Category: {}", test_def.category.as_str()); - info!(" ⏱️ Estimated duration: {:?}", test_def.estimated_duration); - - let test_start = Instant::now(); - let result = self.run_single_test(test_def).await; - let test_duration = test_start.elapsed(); - - match result { - Ok(_) => { - info!("✅ Test passed: {} ({:.2}s)", test_def.name, test_duration.as_secs_f64()); - results.push(TestResult::success(test_def.name.clone(), test_def.category.clone(), test_duration)); - } - Err(e) => { - error!("❌ Test failed: {} ({:.2}s): {}", test_def.name, test_duration.as_secs_f64(), e); - results.push(TestResult::failure( - test_def.name.clone(), - test_def.category.clone(), - test_duration, - e.to_string(), - )); - } - } - - // Add delay between tests to avoid resource conflicts - if i < tests_to_run.len() - 1 { - debug!("⏸️ Waiting two seconds before the next test..."); - sleep(Duration::from_secs(2)).await; - } - } - - let total_duration = start_time.elapsed(); - self.print_test_summary(&results, total_duration); - - results - } - - /// Run a single test by dispatching to the appropriate test function - async fn run_single_test(&self, test_def: &TestDefinition) -> Result<(), Box> { - // This is a placeholder for test dispatch logic - // In a real implementation, this would dispatch to actual test functions - warn!("⚠️ Test '{}' is not implemented in the unified runner; skipping", test_def.name); - Ok(()) - } - - /// Print comprehensive test summary - fn print_test_summary(&self, results: &[TestResult], total_duration: Duration) { - info!("📊 KMS test suite summary"); - info!("⏱️ Total duration: {:.2} seconds", total_duration.as_secs_f64()); - info!("📈 Total tests: {}", results.len()); - - let passed = results.iter().filter(|r| r.success).count(); - let failed = results.iter().filter(|r| !r.success).count(); - - info!("✅ Passed: {}", passed); - info!("❌ Failed: {}", failed); - info!("📊 Success rate: {:.1}%", (passed as f64 / results.len() as f64) * 100.0); - - // Summary by category - let mut category_summary: std::collections::HashMap = std::collections::HashMap::new(); - for result in results { - let (total, passed_count) = category_summary.entry(result.category.clone()).or_insert((0, 0)); - *total += 1; - if result.success { - *passed_count += 1; - } - } - - info!("📊 Category summary:"); - for (category, (total, passed_count)) in category_summary { - info!( - " 🏷️ {}: {}/{} ({:.1}%)", - category.as_str(), - passed_count, - total, - (passed_count as f64 / total as f64) * 100.0 - ); - } - - // List failed tests - if failed > 0 { - warn!("❌ Failing tests:"); - for result in results.iter().filter(|r| !r.success) { - warn!(" - {}: {}", result.test_name, result.error_message.as_deref().unwrap_or("Unknown error")); - } - } - } -} - -/// Quick test suite for critical tests only -#[tokio::test] -async fn test_kms_critical_suite() -> Result<(), Box> { - let config = TestSuiteConfig { - categories: vec![TestCategory::CoreFunctionality, TestCategory::MultipartEncryption], - include_critical_only: true, - max_duration: Some(Duration::from_secs(600)), // 10 minutes max - parallel_execution: false, - }; - - let suite = KMSTestSuite::new().with_config(config); - let results = suite.run_test_suite().await; - - let failed_count = results.iter().filter(|r| !r.success).count(); - if failed_count > 0 { - return Err(format!("Critical test suite failed: {failed_count} tests failed").into()); - } - - info!("✅ All critical tests passed"); - Ok(()) -} - -/// Full comprehensive test suite -#[tokio::test] -async fn test_kms_full_suite() -> Result<(), Box> { - let suite = KMSTestSuite::new(); - let results = suite.run_test_suite().await; - - let total_tests = results.len(); - let failed_count = results.iter().filter(|r| !r.success).count(); - let success_rate = ((total_tests - failed_count) as f64 / total_tests as f64) * 100.0; - - info!("📊 Full suite success rate: {:.1}%", success_rate); - - // Allow up to 10% failure rate for non-critical tests - if success_rate < 90.0 { - return Err(format!("Test suite success rate too low: {success_rate:.1}%").into()); - } - - info!("✅ Full test suite succeeded"); - Ok(()) -} diff --git a/crates/e2e_test/src/object_lambda_test.rs b/crates/e2e_test/src/object_lambda_test.rs index aa6f7d4a1..d2d5f0036 100644 --- a/crates/e2e_test/src/object_lambda_test.rs +++ b/crates/e2e_test/src/object_lambda_test.rs @@ -16,6 +16,7 @@ use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_lo use aws_sdk_s3::primitives::ByteStream; use http::header::{CONTENT_TYPE, HOST}; use reqwest::StatusCode; +use rustfs_config::{ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, ENV_NOTIFY_ENABLE}; use rustfs_signer::pre_sign_v4; use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS; use s3s::Body; @@ -976,7 +977,8 @@ async fn test_get_object_lambda_rejects_disabled_target() -> Result<(), Box Result<(), Box Resul init_logging(); let mut env = RustFSTestEnvironment::new().await?; - env.start_rustfs_server(vec![]).await?; + env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")]) + .await?; let bucket = "object-lambda-e2e-invalid-endpoint"; @@ -1064,7 +1074,8 @@ async fn test_configure_object_lambda_notify_webhook_rejects_response_header_tim init_logging(); let mut env = RustFSTestEnvironment::new().await?; - env.start_rustfs_server(vec![]).await?; + env.start_rustfs_server_with_env(vec![], &[(ENV_NOTIFY_ENABLE, "true")]) + .await?; let response = send_configure_webhook_target_request( &env, @@ -1173,6 +1184,8 @@ async fn test_listen_notification_fans_in_remote_node_events() -> Result<(), Box init_logging(); let mut cluster = RustFSTestClusterEnvironment::new(2).await?; + cluster.set_env(ENV_NOTIFY_ENABLE, "true"); + cluster.set_env(ENV_DRIVE_ACTIVE_CHECK_INTERVAL_SECS, "1"); cluster.start().await?; let bucket = "listen-notification-cluster"; diff --git a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs index c5fc45e17..82d0924bc 100644 --- a/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs +++ b/crates/e2e_test/src/stale_multipart_cleanup_cluster_test.rs @@ -15,7 +15,6 @@ use crate::common::{RustFSTestClusterEnvironment, init_logging}; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::primitives::ByteStream; -use aws_sdk_s3::types::CompletedMultipartUpload; use tokio::time::{Duration, sleep}; use tracing::info; use uuid::Uuid; @@ -43,32 +42,18 @@ async fn list_parts_reports_missing_upload( } } -async fn complete_reports_missing_upload( +async fn multipart_listing_reports_missing_upload( client: &aws_sdk_s3::Client, bucket: &str, key: &str, upload_id: &str, ) -> Result> { - let result = client - .complete_multipart_upload() - .bucket(bucket) - .key(key) - .upload_id(upload_id) - .multipart_upload(CompletedMultipartUpload::builder().build()) - .send() - .await; - match result { - Ok(_) => Ok(false), - Err(SdkError::ServiceError(err)) => { - let code = err.err().meta().code().unwrap_or(""); - if code == "NoSuchUpload" { - Ok(true) - } else { - Err(format!("unexpected complete_multipart_upload service error: code={code}, err={err:?}").into()) - } - } - Err(err) => Err(format!("unexpected complete_multipart_upload error: {err:?}").into()), - } + let result = client.list_multipart_uploads().bucket(bucket).prefix(key).send().await?; + + Ok(!result + .uploads() + .iter() + .any(|upload| upload.key() == Some(key) && upload.upload_id() == Some(upload_id))) } async fn wait_for_cleanup_on_all_nodes( @@ -81,8 +66,8 @@ async fn wait_for_cleanup_on_all_nodes( let mut all_cleaned = true; for (idx, client) in clients.iter().enumerate() { let list_parts_missing = list_parts_reports_missing_upload(client, bucket, key, upload_id).await?; - let complete_missing = complete_reports_missing_upload(client, bucket, key, upload_id).await?; - if !(list_parts_missing && complete_missing) { + let listing_missing = multipart_listing_reports_missing_upload(client, bucket, key, upload_id).await?; + if !(list_parts_missing && listing_missing) { info!("stale multipart still visible on node {} at attempt {}", idx, attempt + 1); all_cleaned = false; break; @@ -146,6 +131,10 @@ async fn test_stale_multipart_cleanup_removes_incomplete_upload_across_cluster() 1, "multipart upload should be visible before background cleanup" ); + assert!( + !multipart_listing_reports_missing_upload(&clients[2], CLEANUP_BUCKET, &key, &upload_id).await?, + "multipart upload listing should contain the upload before background cleanup" + ); wait_for_cleanup_on_all_nodes(&clients, CLEANUP_BUCKET, &key, &upload_id).await?; diff --git a/crates/ecstore/src/cluster/rpc/remote_disk.rs b/crates/ecstore/src/cluster/rpc/remote_disk.rs index 32bf3ae57..a47533a81 100644 --- a/crates/ecstore/src/cluster/rpc/remote_disk.rs +++ b/crates/ecstore/src/cluster/rpc/remote_disk.rs @@ -42,8 +42,9 @@ use futures::lock::Mutex; use metrics::counter; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; use rustfs_io_metrics::internode_metrics::{ - INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, - INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, + INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, + INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE, + INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, }; use rustfs_protos::ChannelClass; use rustfs_protos::evict_failed_connection; @@ -98,6 +99,7 @@ const NS_SCANNER_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5); const REMOTE_DISK_READ_RETRY_BASE_BACKOFF: Duration = Duration::from_millis(50); const ENV_RUSTFS_METADATA_BATCH_READ: &str = "RUSTFS_METADATA_BATCH_READ"; const LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC: &str = "RUSTFS_BATCH_METADATA_RPC"; +const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE"; const BATCH_METADATA_RPC_OFF: &str = "off"; const BATCH_METADATA_RPC_AUTO: &str = "auto"; const BATCH_METADATA_RPC_ON: &str = "on"; @@ -202,7 +204,8 @@ fn parse_batch_metadata_rpc_mode(raw: &str) -> BatchMetadataRpcMode { } fn batch_metadata_rpc_mode_from_env() -> BatchMetadataRpcMode { - rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ) + rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE) + .or_else(|| rustfs_utils::get_env_opt_str(ENV_RUSTFS_METADATA_BATCH_READ)) .or_else(|| rustfs_utils::get_env_opt_str(LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC)) .as_deref() .map(parse_batch_metadata_rpc_mode) @@ -1826,6 +1829,12 @@ fn record_read_version_stage(stage: &'static str, started_at: Option) { } } +fn record_batch_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_stage(stage, started_at.elapsed()); + } +} + /// Aggregate encoded size (bytes) of a `ReadMultiple` response, preferring the msgpack payloads /// and falling back to the JSON compatibility strings. Used to size the RPC for the payload /// histogram / large-payload alerting (grpc-optimization P0 instrumentation). @@ -1936,6 +1945,27 @@ fn decode_batch_read_version_response_items( Ok(batch_read_version_resps) } +fn batch_read_version_request_payload_len(req: &BatchReadVersionReq, req_json: &str, req_bin: &[u8]) -> usize { + req.items + .iter() + .fold(req_json.len().saturating_add(req_bin.len()), |total, item| { + total + .saturating_add(item.org_volume.len()) + .saturating_add(item.volume.len()) + .saturating_add(item.path.len()) + .saturating_add(item.version_id.len()) + }) +} + +fn batch_read_version_response_payload_len(response: &BatchReadVersionResponse) -> usize { + response + .batch_read_version_resps + .iter() + .map(String::len) + .sum::() + .saturating_add(response.batch_read_version_resps_bin.iter().map(Bytes::len).sum::()) +} + fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> { file_info.validate_for_metadata_read().map_err(Into::into) } @@ -2837,14 +2867,19 @@ impl DiskAPI for RemoteDisk { state = "started", "Remote disk RPC started" ); + let batch_read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let encode_started = read_version_stage_timer(batch_read_version_attribution_enabled); let batch_read_version_req = compat_json(&req)?; let batch_read_version_req_bin = encode_msgpack(&req)?; - + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started); + let request_payload_bytes = batch_read_version_attribution_enabled + .then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin)); let batch_result = self .execute_with_timeout_for_op( "batch_read_version", move || async move { let disk = self.disk_ref().await; + let disk_len = disk.len(); let mut client = self .get_bulk_client() .await @@ -2855,9 +2890,20 @@ impl DiskAPI for RemoteDisk { batch_read_version_req_bin: batch_read_version_req_bin.into(), }); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request(); + if let Some(request_payload_bytes) = request_payload_bytes { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes( + request_payload_bytes.saturating_add(disk_len), + ); + } + let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled); let response = match client.batch_read_version(request).await { - Ok(response) => response.into_inner(), + Ok(response) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started); + response.into_inner() + } Err(status) if status.code() == Code::Unimplemented => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started); if mode.should_fallback_on_unimplemented() { record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_FALLBACK_UNIMPLEMENTED); warn!( @@ -2874,6 +2920,7 @@ impl DiskAPI for RemoteDisk { } record_batch_read_version_gate_decision(mode, BATCH_READ_VERSION_GATE_UNSUPPORTED_NO_FALLBACK); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error(); warn!( event = EVENT_REMOTE_DISK_RPC, component = LOG_COMPONENT_ECSTORE, @@ -2886,14 +2933,33 @@ impl DiskAPI for RemoteDisk { ); return Err(Error::from(status)); } - Err(status) => return Err(Error::from(status)), + Err(status) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error(); + return Err(Error::from(status)); + } }; if !response.success { + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error(); return Err(response.error.unwrap_or_default().into()); } - decode_batch_read_version_response_items(response, &self.endpoint).map(Some) + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_recv_bytes( + batch_read_version_response_payload_len(&response), + ); + let decode_started = read_version_stage_timer(batch_read_version_attribution_enabled); + match decode_batch_read_version_response_items(response, &self.endpoint) { + Ok(batch_read_version_resps) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started); + Ok(Some(batch_read_version_resps)) + } + Err(err) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE, decode_started); + crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_error(); + Err(err) + } + } }, get_max_timeout_duration(), ) @@ -4621,6 +4687,7 @@ mod tests { } else { "file version not found".to_string() }, + error_code: if success { 0 } else { DiskError::FileVersionNotFound.to_u32() }, } } @@ -4740,6 +4807,7 @@ mod tests { fn batch_metadata_rpc_mode_uses_documented_env_before_legacy_alias() { temp_env::with_vars( [ + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>), (ENV_RUSTFS_METADATA_BATCH_READ, Some("auto")), (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")), ], @@ -4749,10 +4817,25 @@ mod tests { ); } + #[test] + fn batch_metadata_rpc_mode_uses_get_coalescer_env_before_batch_env() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("on")), + (ENV_RUSTFS_METADATA_BATCH_READ, Some("off")), + (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("off")), + ], + || { + assert_eq!(batch_metadata_rpc_mode_from_env(), BatchMetadataRpcMode::On); + }, + ); + } + #[test] fn batch_metadata_rpc_mode_falls_back_to_legacy_env_alias() { temp_env::with_vars( [ + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, None::<&str>), (ENV_RUSTFS_METADATA_BATCH_READ, None::<&str>), (LEGACY_ENV_RUSTFS_BATCH_METADATA_RPC, Some("on")), ], diff --git a/crates/ecstore/src/cluster/rpc/runtime_sources.rs b/crates/ecstore/src/cluster/rpc/runtime_sources.rs index 0c8393a2e..ab9a34fe9 100644 --- a/crates/ecstore/src/cluster/rpc/runtime_sources.rs +++ b/crates/ecstore/src/cluster/rpc/runtime_sources.rs @@ -14,9 +14,10 @@ use rustfs_io_metrics::internode_metrics::{ INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_RESPONSE, - INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, INTERNODE_OPERATION_GRPC_READ_VERSION, - INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, INTERNODE_OPERATION_READ_FILE_STREAM, - INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, global_internode_metrics, + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE, + INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_OPERATION_PUT_FILE_STREAM, + INTERNODE_OPERATION_READ_FILE_STREAM, INTERNODE_TRANSPORT_BACKEND_GRPC, INTERNODE_TRANSPORT_BACKEND_TCP_HTTP, + global_internode_metrics, }; use std::time::Duration; @@ -93,6 +94,59 @@ pub(crate) fn record_remote_disk_grpc_read_version_request() { ); } +pub(crate) fn record_remote_disk_grpc_batch_read_version_request() { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_outgoing_request_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); +} + +pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static str, duration: Duration) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + stage, + duration, + ); +} + +pub(crate) fn record_remote_disk_grpc_batch_read_version_error() { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics() + .record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC); +} + +pub(crate) fn record_remote_disk_grpc_batch_read_version_sent_bytes(bytes: usize) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_sent_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + bytes, + ); +} + +pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize) { + if !rustfs_io_metrics::get_stage_metrics_enabled() { + return; + } + global_internode_metrics().record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + bytes, + ); + record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes); +} + pub(crate) fn record_remote_disk_grpc_read_version_error() { if !rustfs_io_metrics::get_stage_metrics_enabled() { return; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index 3f75f18a3..d1fdf6778 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -81,7 +81,7 @@ use std::sync::{ atomic::{AtomicBool, AtomicUsize, Ordering}, }; use time::{Duration, OffsetDateTime}; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -100,6 +100,10 @@ const DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF: Duration = Duration::seconds(1); const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY"; const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4; const DECOMMISSION_META_PREFIXES: [&str; 3] = [CONFIG_PREFIX, BUCKET_META_PREFIX, ILM_META_PREFIX]; +const DECOMMISSION_ENTRY_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_ENTRY_CONCURRENCY"; +const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8; +const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64; +const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2; const DECOMMISSION_TARGET_CAPACITY_OVERHEAD_PERCENT: usize = 30; const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3; const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5); @@ -370,6 +374,19 @@ fn decommission_bucket_concurrency_limit() -> usize { rustfs_utils::get_env_usize(DECOMMISSION_BUCKET_CONCURRENCY_ENV, default_limit).max(1) } +fn default_decommission_entry_concurrency(cpu_count: usize) -> usize { + cpu_count.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP) +} + +fn clamp_decommission_entry_concurrency(limit: usize) -> usize { + limit.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP) +} + +fn decommission_entry_concurrency_limit() -> usize { + let default_limit = default_decommission_entry_concurrency(num_cpus::get()); + clamp_decommission_entry_concurrency(rustfs_utils::get_env_usize(DECOMMISSION_ENTRY_CONCURRENCY_ENV, default_limit)) +} + fn is_decommission_meta_bucket(bucket: &DecomBucketInfo) -> bool { bucket.name == RUSTFS_META_BUCKET } @@ -548,6 +565,7 @@ fn spawn_decommission_index_cancelers( store: Arc, rx: CancellationToken, index_cancelers: Vec<(usize, DecommissionCancelerGuard)>, + entry_budget: Arc, ) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { let mut stop_queue = false; @@ -563,7 +581,8 @@ fn spawn_decommission_index_cancelers( let worker = tokio::spawn({ let store = store.clone(); let canceler = canceler.clone(); - async move { store.do_decommission_in_routine(canceler, idx).await } + let entry_budget = entry_budget.clone(); + async move { store.do_decommission_in_routine(canceler, idx, entry_budget).await } }); if let Err(err) = await_decommission_worker(idx, worker).await { error!( @@ -797,6 +816,47 @@ fn count_decommission_item(meta: &mut PoolMeta, idx: usize, size: usize, failed: Ok(()) } +fn ensure_decommission_generation(meta: &PoolMeta, idx: usize, generation: OffsetDateTime) -> Result<()> { + let Some(pool) = meta.pools.get(idx) else { + return Err(invalid_decommission_pool_index_error(meta.pools.len(), idx)); + }; + let Some(info) = pool.decommission.as_ref() else { + return Err(decommission_metadata_not_initialized_error("check decommission generation")); + }; + + if info.start_time == Some(generation) && !info.queued && is_decommission_active(info.complete, info.failed, info.canceled) { + Ok(()) + } else { + Err(Error::OperationCanceled) + } +} + +async fn run_decommission_side_effect( + rx: &CancellationToken, + operation_gate: &Arc>, + operation: F, +) -> Result +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + let _operation_guard = tokio::select! { + biased; + _ = rx.cancelled() => return Err(Error::OperationCanceled), + guard = operation_gate.read() => guard, + }; + + if rx.is_cancelled() { + return Err(Error::OperationCanceled); + } + + let result = operation().await; + if rx.is_cancelled() { + return Err(Error::OperationCanceled); + } + result +} + fn track_decommission_current_object_stage( meta: &mut PoolMeta, idx: usize, @@ -1493,6 +1553,7 @@ async fn wait_decommission_listing_retry(rx: &CancellationToken, delay: std::tim } } +#[cfg(test)] async fn run_decommission_listing_with_retry( rx: CancellationToken, bucket: String, @@ -1500,11 +1561,31 @@ async fn run_decommission_listing_with_retry( pool_idx: usize, set_idx: usize, max_attempts: usize, - mut list: List, + list: List, ) -> Result<()> where List: FnMut(ListCallback) -> ListFuture, ListFuture: std::future::Future>, +{ + run_decommission_listing_with_retry_and_drain(rx, bucket, cb, pool_idx, set_idx, max_attempts, list, || async { false }).await +} + +#[allow(clippy::too_many_arguments)] +async fn run_decommission_listing_with_retry_and_drain( + rx: CancellationToken, + bucket: String, + cb: ListCallback, + pool_idx: usize, + set_idx: usize, + max_attempts: usize, + mut list: List, + mut drain: Drain, +) -> Result<()> +where + List: FnMut(ListCallback) -> ListFuture, + ListFuture: std::future::Future>, + Drain: FnMut() -> DrainFuture, + DrainFuture: std::future::Future, { let max_attempts = max_attempts.max(1); @@ -1536,7 +1617,12 @@ where "Decommission listing started" ); - match list(cb.clone()).await { + let list_result = list(cb.clone()).await; + if drain().await { + return Ok(()); + } + + match list_result { Ok(()) => { debug!( event = EVENT_DECOMMISSION_BUCKET, @@ -1768,6 +1854,7 @@ where Ok(()) } +#[cfg(test)] async fn wait_decommission_worker_drain(workers: &Semaphore, limit: usize) -> Result<()> { let permits = u32::try_from(limit) .map_err(|_| Error::other(format!("decommission worker limit {limit} exceeds semaphore drain capacity")))?; @@ -2441,7 +2528,7 @@ impl PoolMeta { pub fn decommission_failed(&mut self, idx: usize) -> bool { if let Some(stats) = self.pools.get_mut(idx) { if let Some(d) = &stats.decommission { - if !d.failed { + if is_decommission_active(d.complete, d.failed, d.canceled) { stats.last_update = OffsetDateTime::now_utc(); let mut pd = d.clone(); @@ -2489,7 +2576,7 @@ impl PoolMeta { pub fn decommission_complete(&mut self, idx: usize) -> bool { if let Some(stats) = self.pools.get_mut(idx) { if let Some(d) = &stats.decommission { - if !d.complete { + if is_decommission_active(d.complete, d.failed, d.canceled) { stats.last_update = OffsetDateTime::now_utc(); let mut pd = d.clone(); @@ -3142,12 +3229,13 @@ impl ECStore { snapshot.save(self.pools.clone()).await } - async fn save_decommission_progress_checkpoint(&self, idx: usize) -> Result { + async fn save_decommission_progress_checkpoint(&self, idx: usize, generation: OffsetDateTime) -> Result { // Lock order: save gate, then the short pool metadata read/write sections. Peer // reloads are intentionally performed by the caller after both locks are released. let _save_guard = self.pool_meta_save_gate.lock().await; let (snapshot, checkpoint) = { let pool_meta = self.pool_meta.read().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; let Some(checkpoint) = pool_meta.decommission_progress_checkpoint( idx, DECOMMISSION_PROGRESS_SAVE_INTERVAL, @@ -3461,6 +3549,8 @@ impl ECStore { ); } + self.wait_for_decommission_side_effects().await; + if should_save_pool_meta && let Err(err) = self.save_current_pool_meta().await { if let Some(previous_pool_meta) = previous_pool_meta { let mut pool_meta = self.pool_meta.write().await; @@ -3486,6 +3576,22 @@ impl ECStore { ensure_decommission_terminal_operation_supported(self.single_pool(), "clear decommission")?; let _start_guard = self.start_gate.lock().await; + { + let pool_meta = self.pool_meta.read().await; + let pool_count = pool_meta.pools.len(); + ensure_valid_decommission_pool_index(pool_count, idx)?; + let Some(pool) = pool_meta.pools.get(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let (decommission_present, complete, failed, canceled) = pool + .decommission + .as_ref() + .map(|info| (info.has_decommission_state(), info.complete, info.failed, info.canceled)) + .unwrap_or((false, false, false, false)); + ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled)?; + } + self.cancel_decommission_routines_and_wait(&[idx]).await; + let (should_reload_pool_meta, previous_pool_meta) = { let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); @@ -3501,11 +3607,6 @@ impl ECStore { return Err(err); } - { - let mut cancelers = self.decommission_cancelers.write().await; - take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), idx); - } - if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { let stage = format!("clear_decommission for pool {idx}"); resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; @@ -3514,33 +3615,59 @@ impl ECStore { Ok(()) } - async fn promote_queued_decommission(&self, idx: usize) -> Result<()> { - let (changed, previous_pool_meta) = { + async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result { + // Serialize promotion and generation capture with clear/restart transitions. + let (changed, generation, save_error) = { + let _start_guard = self.start_gate.lock().await; let mut pool_meta = self.pool_meta.write().await; - let previous_pool_meta = pool_meta.clone(); + if pool_meta.pools.get(idx).is_none() { + return Err(Error::other("failed to start decommission: target pool was not found")); + } let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx); let promoted = pool_meta.promote_queued_decommission(idx); - (reconciled || promoted, previous_pool_meta) + let changed = reconciled || promoted; + drop(pool_meta); + + let save_error = if changed { + self.save_current_pool_meta().await.err() + } else { + None + }; + + let generation = self.active_decommission_generation(idx).await?; + (changed, generation, save_error) }; - if changed { - if let Err(err) = self.save_current_pool_meta().await { - let mut pool_meta = self.pool_meta.write().await; - rollback_decommission_pool_meta(&mut pool_meta, previous_pool_meta); + if let Some(err) = save_error { + resolve_decommission_terminal_mark_after_error_result( + self.decommission_failed_for_operation(idx, owner).await, + idx, + &err, + )?; + return Err(err); + } + + if changed && let Some(notification_sys) = runtime_sources::notification_sys() { + let stage = format!("promote_queued_decommission for pool {idx}"); + if let Err(err) = + resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()) + { + resolve_decommission_terminal_mark_after_error_result( + self.decommission_failed_for_operation(idx, owner).await, + idx, + &err, + )?; return Err(err); } - if let Some(notification_sys) = runtime_sources::notification_sys() { - let stage = format!("promote_queued_decommission for pool {idx}"); - resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())?; - } } - Ok(()) + Ok(generation) } #[cfg(test)] pub(crate) async fn promote_queued_decommission_for_test(&self, idx: usize) -> Result<()> { - self.promote_queued_decommission(idx).await + let owner = DecommissionCanceler::new(CancellationToken::new()); + self.promote_queued_decommission(idx, &owner).await.map(|_| ()) } async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> { @@ -3584,6 +3711,21 @@ impl ECStore { is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx)) } + async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) { + { + let mut cancelers = self.decommission_cancelers.write().await; + for idx in indices { + take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx); + } + } + self.wait_for_decommission_side_effects().await; + } + + async fn wait_for_decommission_side_effects(&self) { + let operation_gate = self.ctx.decommission_operation_gate(); + let _operation_guard = operation_gate.write().await; + } + async fn reserve_decommission_routines( &self, rx: &CancellationToken, @@ -3628,7 +3770,12 @@ impl ECStore { ) -> Result<()> { let index_cancelers = self.reserve_decommission_routines(&rx, indices.as_slice()).await?; if !index_cancelers.is_empty() { - std::mem::drop(spawn_decommission_index_cancelers(store, rx, index_cancelers)); + std::mem::drop(spawn_decommission_index_cancelers( + store, + rx, + index_cancelers, + Arc::new(Semaphore::new(decommission_entry_concurrency_limit())), + )); } Ok(()) @@ -3650,7 +3797,12 @@ impl ECStore { return Ok(()); } - std::mem::drop(spawn_decommission_index_cancelers(self.clone(), rx, index_cancelers)); + std::mem::drop(spawn_decommission_index_cancelers( + self.clone(), + rx, + index_cancelers, + Arc::new(Semaphore::new(decommission_entry_concurrency_limit())), + )); Ok(()) } @@ -3675,20 +3827,344 @@ impl ECStore { let index_cancelers = self .start_decommission_with_routines(indices, &rx, local_indices.as_slice()) .await?; - std::mem::drop(spawn_decommission_index_cancelers(store, rx, index_cancelers)); + std::mem::drop(spawn_decommission_index_cancelers( + store, + rx, + index_cancelers, + Arc::new(Semaphore::new(decommission_entry_concurrency_limit())), + )); Ok(()) } + async fn active_decommission_generation(&self, idx: usize) -> Result { + let pool_meta = self.pool_meta.read().await; + let Some(pool) = pool_meta.pools.get(idx) else { + return Err(invalid_decommission_pool_index_error(pool_meta.pools.len(), idx)); + }; + let Some(info) = pool.decommission.as_ref() else { + return Err(decommission_metadata_not_initialized_error("load decommission generation")); + }; + let Some(generation) = info.start_time else { + return Err(Error::OperationCanceled); + }; + ensure_decommission_generation(&pool_meta, idx, generation)?; + Ok(generation) + } + + async fn ensure_decommission_generation_current(&self, idx: usize, generation: OffsetDateTime) -> Result<()> { + let pool_meta = self.pool_meta.read().await; + ensure_decommission_generation(&pool_meta, idx, generation) + } + + #[allow(clippy::too_many_arguments)] + async fn decommission_entry_worker( + self: Arc, + rx: CancellationToken, + idx: usize, + set_idx: usize, + generation: OffsetDateTime, + bucket: String, + set: Arc, + lifecycle_config: Option, + object_lock_config: Option, + replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, + expected_bucket_incarnation_id: Option, + entry_budget: Arc, + queue: Arc>>, + entry_error: Arc>>, + ) { + loop { + let queued = tokio::select! { + biased; + _ = rx.cancelled() => return, + item = async { + let mut queue = queue.lock().await; + queue.recv().await + } => item, + }; + let Some(QueuedDecommissionEntry { entry, queue_permit }) = queued else { + return; + }; + let object_name = entry.name.clone(); + + if entry_error.lock().await.is_some() { + drop(queue_permit); + continue; + } + + if let Err(err) = self.ensure_decommission_generation_current(idx, generation).await { + if matches!(err, Error::OperationCanceled) { + rx.cancel(); + } else { + record_decommission_entry_error(&entry_error, &rx, err).await; + } + return; + } + + if let Err(err) = backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &rx).await + { + if matches!(err, Error::OperationCanceled) { + return; + } + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bucket, + object = %object_name, + state = "entry_admission_failed", + error = %err, + "Decommission entry admission failed" + ); + record_decommission_entry_error(&entry_error, &rx, err).await; + return; + } + + let entry_budget_permit = match tokio::select! { + biased; + _ = rx.cancelled() => return, + permit = entry_budget.clone().acquire_owned() => permit, + } { + Ok(permit) => permit, + Err(err) => { + let err = Error::other(format!("decommission entry budget permit acquire failed: {err}")); + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bucket, + object = %object_name, + state = "entry_budget_acquire_failed", + error = %err, + "Decommission entry budget permit acquire failed" + ); + record_decommission_entry_error(&entry_error, &rx, err).await; + return; + } + }; + + let result = self + .decommission_entry( + rx.clone(), + idx, + generation, + entry, + bucket.clone(), + set.clone(), + lifecycle_config.clone(), + object_lock_config.clone(), + replication_config.clone(), + expected_bucket_incarnation_id, + ) + .await; + drop(entry_budget_permit); + drop(queue_permit); + + if let Err(err) = result { + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bucket, + object = %object_name, + state = "entry_failed", + error = %err, + "Decommission entry failed" + ); + record_decommission_entry_error(&entry_error, &rx, err).await; + return; + } + } + } + + #[allow(clippy::too_many_arguments)] + async fn decommission_set( + self: Arc, + rx: CancellationToken, + idx: usize, + set_idx: usize, + generation: OffsetDateTime, + set: Arc, + bi: DecomBucketInfo, + lifecycle_config: Option, + object_lock_config: Option, + replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, + expected_bucket_incarnation_id: Option, + entry_budget: Arc, + entry_error: Arc>>, + ) -> Result<()> { + let worker_count = DECOMMISSION_ENTRY_WORKERS_PER_SET; + let queue_capacity = decommission_entry_queue_capacity(worker_count); + let outstanding_capacity = queue_capacity.saturating_add(worker_count); + let outstanding = Arc::new(Semaphore::new(outstanding_capacity)); + let (tx, rx_queue) = mpsc::channel(queue_capacity); + let queue = Arc::new(tokio::sync::Mutex::new(rx_queue)); + + let mut entry_workers = tokio::task::JoinSet::new(); + for _ in 0..worker_count { + let this = self.clone(); + let rx = rx.clone(); + let bucket = bi.name.clone(); + let set = set.clone(); + let lifecycle_config = lifecycle_config.clone(); + let object_lock_config = object_lock_config.clone(); + let replication_config = replication_config.clone(); + let queue = queue.clone(); + let entry_budget = entry_budget.clone(); + let entry_error = entry_error.clone(); + entry_workers.spawn(async move { + this.decommission_entry_worker( + rx, + idx, + set_idx, + generation, + bucket, + set, + lifecycle_config, + object_lock_config, + replication_config, + expected_bucket_incarnation_id, + entry_budget, + queue, + entry_error, + ) + .await; + }); + } + + let callback: ListCallback = Arc::new({ + let tx = tx.clone(); + let outstanding = outstanding.clone(); + let callback_rx = rx.clone(); + let entry_error = entry_error.clone(); + let bucket = bi.name.clone(); + move |entry: MetaCacheEntry| { + let tx = tx.clone(); + let outstanding = outstanding.clone(); + let callback_rx = callback_rx.clone(); + let entry_error = entry_error.clone(); + let bucket = bucket.clone(); + Box::pin(async move { + if callback_rx.is_cancelled() || entry_error.lock().await.is_some() { + return; + } + + if matches!( + enqueue_decommission_entry(&callback_rx, &outstanding, &tx, entry).await, + DecommissionEntryEnqueueResult::Closed + ) { + let err = Error::other("decommission entry queue closed"); + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bucket, + state = "entry_queue_closed", + error = %err, + "Decommission entry queue closed" + ); + record_decommission_entry_error(&entry_error, &callback_rx, err).await; + } + }) + } + }); + + let list_set = set.clone(); + let list_rx = rx.clone(); + let list_rx_for_list = list_rx.clone(); + let list_rx_for_drain = list_rx.clone(); + let list_bi = bi.clone(); + let list_outstanding = outstanding.clone(); + let list_entry_error = entry_error.clone(); + let mut listing = tokio::spawn(async move { + run_decommission_listing_with_retry_and_drain( + list_rx.clone(), + list_bi.name.clone(), + callback, + idx, + set_idx, + DECOMMISSION_LISTING_MAX_ATTEMPTS, + move |callback| { + let set = list_set.clone(); + let rx = list_rx_for_list.clone(); + let bucket = list_bi.clone(); + let entry_error = list_entry_error.clone(); + async move { + set.list_objects_to_decommission(rx, bucket, callback, entry_error, idx, set_idx) + .await + } + }, + move || { + let rx = list_rx_for_drain.clone(); + let outstanding = list_outstanding.clone(); + async move { drain_decommission_entry_queue(&rx, &outstanding, outstanding_capacity).await } + }, + ) + .await + }); + + let mut listing_result = None; + let mut workers_left = worker_count; + let mut sender = Some(tx); + while listing_result.is_none() || workers_left > 0 { + tokio::select! { + biased; + result = &mut listing, if listing_result.is_none() => { + let result = resolve_decommission_listing_worker_result(set_idx, result); + if result.is_err() { + rx.cancel(); + } + listing_result = Some(result); + drop(sender.take()); + } + worker_result = entry_workers.join_next(), if workers_left > 0 => { + workers_left -= 1; + if let Some(Err(err)) = worker_result { + let err = Error::other(format!("decommission entry worker {set_idx} task join error: {err}")); + error!( + event = EVENT_DECOMMISSION_ENTRY, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_POOLS, + pool_index = idx, + set_index = set_idx, + bucket = %bi.name, + state = "entry_worker_join_failed", + error = %err, + "Decommission entry worker task failed" + ); + record_decommission_entry_error(&entry_error, &rx, err).await; + } + } + } + } + + let listing_result = listing_result.unwrap_or_else(|| Err(Error::other("decommission listing task did not complete"))); + if let Some(err) = entry_error.lock().await.clone() { + return Err(err); + } + listing_result + } + async fn track_decommission_entry_progress_stage( &self, idx: usize, + generation: OffsetDateTime, bucket: &str, object: &str, stage: &'static str, ) -> Result<()> { { let mut pool_meta = self.pool_meta.write().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; track_decommission_current_object_stage(&mut pool_meta, idx, bucket, object, stage) .map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?; } @@ -3697,15 +4173,15 @@ impl ECStore { } #[allow(unused_assignments, clippy::too_many_arguments)] - #[tracing::instrument(skip(self, set, _worker_permit, lifecycle_config, object_lock_config, replication_config))] + #[tracing::instrument(skip(self, set, lifecycle_config, object_lock_config, replication_config))] async fn decommission_entry( self: &Arc, rx: CancellationToken, idx: usize, + generation: OffsetDateTime, entry: MetaCacheEntry, bucket: String, set: Arc, - _worker_permit: OwnedSemaphorePermit, lifecycle_config: Option, object_lock_config: Option, replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>, @@ -3744,6 +4220,8 @@ impl ECStore { rx.cancel(); } decommission_cancel_signal_result(rx.is_cancelled())?; + self.ensure_decommission_generation_current(idx, generation).await?; + let operation_gate = self.ctx.decommission_operation_gate(); let bucket_incarnation_fence = match expected_bucket_incarnation_id { Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?), @@ -3765,15 +4243,18 @@ impl ECStore { } decommission_cancel_signal_result(rx.is_cancelled())?; - if should_skip_lifecycle_for_data_movement( - self.clone(), - &bucket, - version, - lifecycle_config.as_ref(), - object_lock_config.as_ref(), - true, - &LcEventSrc::Decom, - ) + if run_decommission_side_effect(&rx, &operation_gate, || async { + should_skip_lifecycle_for_data_movement( + self.clone(), + &bucket, + version, + lifecycle_config.as_ref(), + object_lock_config.as_ref(), + true, + &LcEventSrc::Decom, + ) + .await + }) .await .map_err(|err| with_decommission_entry_context("lifecycle_expiry", bucket.as_str(), version.name.as_str(), err))? { @@ -3806,13 +4287,15 @@ impl ECStore { let mut failure = false; let mut error = None; if version.deleted { - if let Err(err) = self - .delete_object( + if let Err(err) = run_decommission_side_effect(&rx, &operation_gate, || async { + self.delete_object( bucket.as_str(), &version.name, decommission_delete_marker_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id), ) .await + }) + .await { if is_decommission_copy_cleanup_safe_error(&err) { warn!( @@ -3864,6 +4347,7 @@ impl ECStore { { let mut pool_meta = self.pool_meta.write().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, failure) { return Err(with_decommission_entry_context( "count_decommission_item", @@ -3895,14 +4379,16 @@ impl ECStore { for _i in 0..3 { if version.is_remote() { - if let Err(err) = self - .decommission_tiered_object( + if let Err(err) = run_decommission_side_effect(&rx, &operation_gate, || async { + self.decommission_tiered_object( bucket.as_str(), &version.name, version, &decommission_remote_tiered_opts(version, version_id.clone(), idx, expected_bucket_incarnation_id), ) .await + }) + .await { if is_decommission_copy_cleanup_safe_error(&err) { ignore = true; @@ -3966,16 +4452,19 @@ impl ECStore { self.track_decommission_entry_progress_stage( idx, + generation, bucket_name.as_str(), object_name.as_str(), DECOMMISSION_STAGE_MIGRATE_OBJECT, ) .await?; - if let Err(err) = self - .clone() - .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id) - .await + if let Err(err) = run_decommission_side_effect(&rx, &operation_gate, || async { + self.clone() + .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id) + .await + }) + .await { if is_decommission_copy_cleanup_safe_error(&err) { ignore = true; @@ -4033,6 +4522,7 @@ impl ECStore { { let mut pool_meta = self.pool_meta.write().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; if let Err(err) = count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), failure) { return Err(with_decommission_entry_context( "count_decommission_item", @@ -4058,9 +4548,11 @@ impl ECStore { return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup")); } decommission_cancel_signal_result(rx.is_cancelled())?; + self.ensure_decommission_generation_current(idx, generation).await?; self.track_decommission_entry_progress_stage( idx, + generation, bucket.as_str(), entry.name.as_str(), DECOMMISSION_STAGE_CLEANUP_PREFLIGHT, @@ -4069,34 +4561,42 @@ impl ECStore { self.track_decommission_entry_progress_stage( idx, + generation, bucket.as_str(), entry.name.as_str(), DECOMMISSION_STAGE_SOURCE_CLEANUP, ) .await?; - let cleanup_result = data_movement::cleanup_source_entry_if_unchanged( - set.clone(), - bucket.as_str(), - entry.name.as_str(), - &fivs, - &cleanup_preflight_allowed_missing, - data_movement::SourceCleanupBucketFence { - expected_incarnation_id: expected_bucket_incarnation_id, - lifecycle_guard: bucket_incarnation_fence - .as_ref() - .and_then(|guard| guard.namespace_lock_guard()), - }, - "decommission", - ) - .await - .map_err(|err| match err { - data_movement::SourceCleanupError::SourceChanged => Error::other(format!( - "decommission: source cleanup preflight failed for {}/{}: source versions changed after migration started", - bucket, entry.name - )), - data_movement::SourceCleanupError::Storage(err) => err, - }); + let source_cleanup_mutation_fence = self + .acquire_decommission_source_cleanup_fence(bucket.as_str(), entry.name.as_str(), set.as_ref()) + .await?; + let cleanup_result = run_decommission_side_effect(&rx, &operation_gate, || async { + data_movement::cleanup_source_entry_if_unchanged( + set.clone(), + bucket.as_str(), + entry.name.as_str(), + &fivs, + &cleanup_preflight_allowed_missing, + data_movement::SourceCleanupBucketFence { + expected_incarnation_id: expected_bucket_incarnation_id, + lifecycle_guard: bucket_incarnation_fence + .as_ref() + .and_then(|guard| guard.namespace_lock_guard()), + object_mutation_fence: Some(&source_cleanup_mutation_fence), + }, + "decommission", + ) + .await + .map_err(|err| match err { + data_movement::SourceCleanupError::SourceChanged => Error::other(format!( + "decommission: source cleanup preflight failed for {}/{}: source versions changed after migration started", + bucket, entry.name + )), + data_movement::SourceCleanupError::Storage(err) => err, + }) + }) + .await; resolve_decommission_entry_cleanup_delete_result(cleanup_result, bucket.as_str(), entry.name.as_str())? } else if durable_ilm_record.is_some() { debug!( @@ -4127,6 +4627,7 @@ impl ECStore { let should_save_progress = { let mut pool_meta = self.pool_meta.write().await; + ensure_decommission_generation(&pool_meta, idx, generation)?; if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) { return Err(with_decommission_entry_context( @@ -4147,6 +4648,7 @@ impl ECStore { self.track_decommission_entry_progress_stage( idx, + generation, bucket.as_str(), entry.name.as_str(), DECOMMISSION_STAGE_ENTRY_FINISHED, @@ -4154,7 +4656,7 @@ impl ECStore { .await?; if should_save_progress { - match self.save_decommission_progress_checkpoint(idx).await { + match self.save_decommission_progress_checkpoint(idx, generation).await { Ok(true) => { if let Some(notification_sys) = runtime_sources::notification_sys() && let Err(err) = resolve_decommission_entry_reload_result( @@ -4198,6 +4700,29 @@ impl ECStore { Ok(()) } + #[cfg(test)] + pub(crate) async fn decommission_entry_for_test( + self: &Arc, + idx: usize, + entry: MetaCacheEntry, + bucket: String, + set: Arc, + ) -> Result<()> { + self.decommission_entry( + CancellationToken::new(), + idx, + OffsetDateTime::now_utc(), + entry, + bucket, + set, + None, + None, + None, + None, + ) + .await + } + #[tracing::instrument(skip(self, rx))] async fn decommission_pool( self: &Arc, @@ -4205,13 +4730,10 @@ impl ECStore { idx: usize, pool: Arc, bi: DecomBucketInfo, + entry_budget: Arc, ) -> Result<()> { - let worker_limit = pool.disk_set.len() * 2; - if worker_limit == 0 { - return Err(Error::other("decommission worker limit must be greater than zero")); - } - let workers = Arc::new(Semaphore::new(worker_limit)); let entry_error = Arc::new(tokio::sync::Mutex::new(None::)); + let generation = self.active_decommission_generation(idx).await?; let mut listing_workers = Vec::with_capacity(pool.disk_set.len()); let mut lifecycle_config = None; @@ -4240,12 +4762,6 @@ impl ECStore { } for (set_idx, set) in pool.disk_set.iter().enumerate() { - let listing_permit = workers - .clone() - .acquire_owned() - .await - .map_err(|err| Error::other(format!("decommission listing worker permit acquire failed: {err}")))?; - debug!( event = EVENT_DECOMMISSION_BUCKET, component = LOG_COMPONENT_ECSTORE, @@ -4257,130 +4773,34 @@ impl ECStore { "Decommission listing worker started" ); - let decommission_entry: ListCallback = Arc::new({ - let this = Arc::clone(self); - let bucket = bi.name.clone(); - let workers = workers.clone(); - let set = set.clone(); - let lifecycle_config = lifecycle_config.clone(); - let object_lock_config = object_lock_config.clone(); - let replication_config = replication_config.clone(); - let entry_error = entry_error.clone(); - let callback_rx = rx.clone(); - move |entry: MetaCacheEntry| { - let this = this.clone(); - let bucket = bucket.clone(); - let workers = workers.clone(); - let set = set.clone(); - let lifecycle_config = lifecycle_config.clone(); - let object_lock_config = object_lock_config.clone(); - let replication_config = replication_config.clone(); - let expected_bucket_incarnation_id = expected_bucket_incarnation_id; - let entry_error = entry_error.clone(); - let callback_rx = callback_rx.clone(); - - Box::pin(async move { - if callback_rx.is_cancelled() { - return; - } - if entry_error.lock().await.is_some() { - return; - } - - if let Err(err) = - backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &callback_rx) - .await - { - if matches!(err, Error::OperationCanceled) { - return; - } - error!("decommission_pool: data movement admission failed: {err}"); - let mut first_err = entry_error.lock().await; - if first_err.is_none() { - *first_err = Some(err); - callback_rx.cancel(); - } - return; - } - - if entry_error.lock().await.is_some() { - return; - } - - let worker_permit = match tokio::select! { - _ = callback_rx.cancelled() => return, - permit = workers.clone().acquire_owned() => permit, - } { - Ok(permit) => permit, - Err(err) => { - let err = Error::other(format!("decommission entry worker permit acquire failed: {err}")); - error!("decommission_pool: decommission_entry failed: {err}"); - let mut first_err = entry_error.lock().await; - if first_err.is_none() { - *first_err = Some(err); - callback_rx.cancel(); - } - return; - } - }; - if entry_error.lock().await.is_some() { - return; - } - let entry_rx = callback_rx.clone(); - if let Err(err) = this - .decommission_entry( - entry_rx, - idx, - entry, - bucket, - set, - worker_permit, - lifecycle_config, - object_lock_config, - replication_config, - expected_bucket_incarnation_id, - ) - .await - { - error!("decommission_pool: decommission_entry failed: {err}"); - let mut first_err = entry_error.lock().await; - if first_err.is_none() { - *first_err = Some(err); - callback_rx.cancel(); - } - } - }) - } - }); - let set = set.clone(); + let store = Arc::clone(self); let rx_clone = rx.clone(); - let bi = bi.clone(); - let set_id = set_idx; - let listing_entry_error = entry_error.clone(); + let bi_clone = bi.clone(); + let lifecycle_config = lifecycle_config.clone(); + let object_lock_config = object_lock_config.clone(); + let replication_config = replication_config.clone(); + let entry_budget = entry_budget.clone(); + let entry_error = entry_error.clone(); let worker = tokio::spawn(async move { - let _listing_permit = listing_permit; - run_decommission_listing_with_retry( - rx_clone.clone(), - bi.name.clone(), - decommission_entry.clone(), - idx, - set_id, - DECOMMISSION_LISTING_MAX_ATTEMPTS, - |callback| { - let set = set.clone(); - let rx = rx_clone.clone(); - let bucket = bi.clone(); - let entry_error = listing_entry_error.clone(); - async move { - set.list_objects_to_decommission(rx, bucket, callback, entry_error.clone(), idx, set_id) - .await - } - }, - ) - .await + store + .decommission_set( + rx_clone, + idx, + set_idx, + generation, + set, + bi_clone, + lifecycle_config, + object_lock_config, + replication_config, + expected_bucket_incarnation_id, + entry_budget, + entry_error, + ) + .await }); - listing_workers.push((set_id, worker)); + listing_workers.push((set_idx, worker)); } debug!( @@ -4403,9 +4823,11 @@ impl ECStore { } } - wait_decommission_worker_drain(&workers, worker_limit).await?; + if let Some(err) = listing_worker_error { + return Err(err); + } - if let Some(err) = resolve_decommission_listing_error(listing_worker_error, entry_error.lock().await.clone()) { + if let Some(err) = entry_error.lock().await.clone() { return Err(err); } @@ -4448,9 +4870,14 @@ impl ECStore { } #[tracing::instrument(skip(self, canceler))] - pub async fn do_decommission_in_routine(self: &Arc, canceler: DecommissionCanceler, idx: usize) -> Result<()> { + pub async fn do_decommission_in_routine( + self: &Arc, + canceler: DecommissionCanceler, + idx: usize, + entry_budget: Arc, + ) -> Result<()> { let rx = canceler.token().clone(); - self.run_decommission_in_routine(rx, idx, &canceler).await + self.run_decommission_in_routine(rx, idx, &canceler, entry_budget).await } async fn run_decommission_in_routine( @@ -4458,15 +4885,20 @@ impl ECStore { rx: CancellationToken, idx: usize, canceler: &DecommissionCanceler, + entry_budget: Arc, ) -> Result<()> { - if let Err(err) = self.promote_queued_decommission(idx).await { - resolve_decommission_terminal_mark_after_error_result( - self.decommission_failed_for_operation(idx, canceler).await, - idx, - &err, - )?; - return Err(err); - } + let generation = match self.promote_queued_decommission(idx, canceler).await { + Ok(generation) => generation, + Err(Error::OperationCanceled) => return Ok(()), + Err(err) => { + resolve_decommission_terminal_mark_after_error_result( + self.decommission_failed_for_operation(idx, canceler).await, + idx, + &err, + )?; + return Err(err); + } + }; if rx.is_cancelled() { let already_canceled = { let pool_meta = self.pool_meta.read().await; @@ -4493,7 +4925,7 @@ impl ECStore { } return Ok(()); } - let result = self.decommission_in_background(rx.clone(), idx).await; + let result = self.decommission_in_background(rx.clone(), idx, entry_budget).await; let (final_state, canceled, cmd_line) = { let pool_meta = self.pool_meta.read().await; @@ -4607,6 +5039,11 @@ impl ECStore { "failed to finalize decommission for pool {cmd_line}: post-check failed: {err}" ))); } + if self.decommission_cancel_requested(idx, &rx).await { + rx.cancel(); + } + decommission_cancel_signal_result(rx.is_cancelled())?; + self.ensure_decommission_generation_current(idx, generation).await?; info!( event = EVENT_DECOMMISSION_STATE, component = LOG_COMPONENT_ECSTORE, @@ -4871,6 +5308,7 @@ impl ECStore { idx: usize, pool: Arc, bucket: DecomBucketInfo, + entry_budget: Arc, ) -> Result<()> { let is_decommissioned = { let pool_meta = self.pool_meta.read().await; @@ -4896,7 +5334,10 @@ impl ECStore { warn!("decommission: currently on bucket {}", &bucket.name); - if let Err(err) = self.decommission_pool(rx.clone(), idx, pool, bucket.clone()).await { + if let Err(err) = self + .decommission_pool(rx.clone(), idx, pool, bucket.clone(), entry_budget) + .await + { error!("decommission: decommission_pool err {:?}", &err); return Err(err); } else { @@ -4930,40 +5371,53 @@ impl ECStore { pool: Arc, buckets: Vec, limit: usize, + entry_budget: Arc, ) -> Result<()> { let store = Arc::clone(self); run_decommission_buckets_bounded(rx, buckets, limit, move |bucket, rx| { let store = Arc::clone(&store); let pool = pool.clone(); - Box::pin(async move { store.decommission_pending_bucket(rx, idx, pool, bucket).await }) + let entry_budget = entry_budget.clone(); + Box::pin(async move { store.decommission_pending_bucket(rx, idx, pool, bucket, entry_budget).await }) }) .await } #[tracing::instrument(skip(self, rx))] - async fn decommission_in_background(self: &Arc, rx: CancellationToken, idx: usize) -> Result<()> { + async fn decommission_in_background( + self: &Arc, + rx: CancellationToken, + idx: usize, + entry_budget: Arc, + ) -> Result<()> { let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone(); let pending = { let pool_meta = self.pool_meta.read().await; pool_meta.pending_buckets(idx) }; - let bucket_concurrency = decommission_bucket_concurrency_limit(); if bucket_concurrency <= 1 { for bucket in pending { - self.decommission_pending_bucket(rx.clone(), idx, pool.clone(), bucket) + self.decommission_pending_bucket(rx.clone(), idx, pool.clone(), bucket, entry_budget.clone()) .await?; } return Ok(()); } let (regular_buckets, meta_buckets) = split_decommission_buckets(pending); - self.decommission_buckets_concurrently(rx.clone(), idx, pool.clone(), regular_buckets, bucket_concurrency) - .await?; + self.decommission_buckets_concurrently( + rx.clone(), + idx, + pool.clone(), + regular_buckets, + bucket_concurrency, + entry_budget.clone(), + ) + .await?; for bucket in meta_buckets { - self.decommission_pending_bucket(rx.clone(), idx, pool.clone(), bucket) + self.decommission_pending_bucket(rx.clone(), idx, pool.clone(), bucket, entry_budget.clone()) .await?; } @@ -5030,6 +5484,8 @@ impl ECStore { self.ensure_decommission_rebalance_idle_after_refresh().await?; let all_space_infos = self.get_decommission_all_pool_space_infos().await?; + self.cancel_decommission_routines_and_wait(&indices).await; + let index_cancelers = if let Some((rx, local_indices)) = reservation { // Lock order matches terminal transitions: decommission_cancelers // before pool_meta while start_gate excludes another start. @@ -6256,15 +6712,20 @@ impl ECStore { ) -> Result<()> { warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name); let object_name = rd.object_info.name.clone(); - let result = data_movement::migrate_object( + let mut migration = tokio::task::JoinSet::new(); + migration.spawn(data_movement::migrate_decommission_object( self, pool_idx, bucket.clone(), rd, expected_bucket_incarnation_id, "decommission_object", - ) - .await; + )); + let result = migration + .join_next() + .await + .ok_or_else(|| Error::other("decommission migration task was not started"))? + .map_err(|err| Error::other(format!("decommission migration task join error: {err}")))?; if result.is_ok() { warn!("decommission_object: migrated {} {}", &bucket, &object_name); } @@ -6579,6 +7040,14 @@ mod tests { let mut pool_meta = build_pool_meta(); assert!(pool_meta.decommission_cancel(0)); assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None); + + let mut pool_meta = build_pool_meta(); + assert!(pool_meta.decommission_cancel(0)); + assert!(!pool_meta.decommission_complete(0)); + + let mut pool_meta = build_pool_meta(); + assert!(pool_meta.decommission_failed(0)); + assert!(!pool_meta.decommission_complete(0)); } #[test] @@ -6962,6 +7431,79 @@ mod tests { pub type ListCallback = Arc BoxFuture<'static, ()> + Send + Sync + 'static>; +const DECOMMISSION_ENTRY_QUEUE_HARD_CAP: usize = 256; + +struct QueuedDecommissionEntry { + entry: MetaCacheEntry, + queue_permit: OwnedSemaphorePermit, +} + +enum DecommissionEntryEnqueueResult { + Enqueued, + Canceled, + Closed, +} + +fn decommission_entry_queue_capacity(worker_limit: usize) -> usize { + worker_limit.saturating_mul(2).clamp(1, DECOMMISSION_ENTRY_QUEUE_HARD_CAP) +} + +async fn enqueue_decommission_entry( + rx: &CancellationToken, + outstanding: &Arc, + tx: &mpsc::Sender, + entry: MetaCacheEntry, +) -> DecommissionEntryEnqueueResult { + let queue_permit = match tokio::select! { + biased; + _ = rx.cancelled() => return DecommissionEntryEnqueueResult::Canceled, + permit = outstanding.clone().acquire_owned() => permit, + } { + Ok(permit) => permit, + Err(_) => return DecommissionEntryEnqueueResult::Closed, + }; + + let queued = QueuedDecommissionEntry { entry, queue_permit }; + tokio::select! { + biased; + _ = rx.cancelled() => DecommissionEntryEnqueueResult::Canceled, + result = tx.send(queued) => { + if result.is_ok() { + DecommissionEntryEnqueueResult::Enqueued + } else { + DecommissionEntryEnqueueResult::Closed + } + } + } +} + +async fn drain_decommission_entry_queue(rx: &CancellationToken, outstanding: &Arc, capacity: usize) -> bool { + let Ok(permits) = u32::try_from(capacity) else { + return true; + }; + + tokio::select! { + _ = rx.cancelled() => true, + result = outstanding.acquire_many(permits) => result.is_err(), + } +} + +async fn record_decommission_entry_error( + entry_error: &Arc>>, + rx: &CancellationToken, + err: Error, +) { + if rx.is_cancelled() { + return; + } + + let mut first_err = entry_error.lock().await; + if first_err.is_none() && !rx.is_cancelled() { + *first_err = Some(err); + rx.cancel(); + } +} + impl SetDisks { #[tracing::instrument(skip(self, rx, cb_func, entry_error))] async fn list_objects_to_decommission( @@ -7253,34 +7795,39 @@ mod pools_tests { use super::record_decommission_entry_error; use super::resolve_decommission_listing_error; use super::{ - DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, DECOMMISSION_META_PREFIXES, DECOMMISSION_PROGRESS_SAVE_INTERVAL, - DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo, DecommissionCanceler, DecommissionDurableIlmReceipt, - DecommissionStartPoolState, DecommissionTerminalState, ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, - PoolStatus, apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers, - bind_missing_decommission_cancelers, cancel_decommission_canceler, classify_decommission_terminal_state, - count_decommission_item, decommission_cancel_signal_result, decommission_durable_ilm_receipt_path, - decommission_durable_ilm_receipt_run_prefix, decommission_durable_ilm_receipt_run_token, decommission_item_size, + DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, + DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP, DECOMMISSION_META_PREFIXES, + DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD, DecomBucketInfo, DecommissionCanceler, + DecommissionDurableIlmReceipt, DecommissionEntryEnqueueResult, DecommissionStartPoolState, DecommissionTerminalState, + ListCallback, PoolDecommissionInfo, PoolMeta, PoolSpaceInfo, PoolStatus, QueuedDecommissionEntry, + apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers, + bind_missing_decommission_cancelers, cancel_decommission_canceler, clamp_decommission_entry_concurrency, + classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result, + decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix, + decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size, decommission_meta_bucket_options, decommission_start_pool_state, dedup_indices, default_decommission_bucket_concurrency, - ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_listing_disks_available, - ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool, - ensure_decommission_start_local_leader, ensure_decommission_start_pool_states, - ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity, - ensure_decommission_terminal_operation_supported, ensure_local_decommission_pool_leaders, - ensure_valid_decommission_pool_index, first_resumable_decommission_queue_indices, get_by_index, - guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested, - load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done, - merge_decommission_durable_ilm_receipts, merge_pool_status_refresh, missing_decommission_worker_prefix, - observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, reconcile_decommission_meta_buckets, - require_decommission_store, reserve_decommission_start_cancelers, resolve_decommission_bucket_done_save_result, - resolve_decommission_bucket_state, resolve_decommission_check_after_list_result, - resolve_decommission_entry_cleanup_delete_result, resolve_decommission_entry_exact_versions, - resolve_decommission_entry_reload_result, resolve_decommission_listing_worker_result, - resolve_decommission_optional_bucket_config_result, resolve_decommission_partial_listing_entry, - resolve_decommission_pool_meta_reload_result, resolve_decommission_preflight_heal_result, - resolve_decommission_progress_save_result, resolve_decommission_terminal_mark_after_error_result, - resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result, - resolve_start_decommission_pool_meta_reload_result, rollback_start_decommission_pool_meta, - run_decommission_buckets_bounded, run_decommission_listing_with_retry, should_cleanup_decommission_source_entry, + default_decommission_entry_concurrency, drain_decommission_entry_queue, enqueue_decommission_entry, + ensure_decommission_cancel_allowed, ensure_decommission_clear_allowed, ensure_decommission_generation, + ensure_decommission_listing_disks_available, ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, + ensure_decommission_start_keeps_active_pool, ensure_decommission_start_local_leader, + ensure_decommission_start_pool_states, ensure_decommission_start_rebalance_meta_allowed, + ensure_decommission_start_target_capacity, ensure_decommission_terminal_operation_supported, + ensure_local_decommission_pool_leaders, ensure_valid_decommission_pool_index, first_resumable_decommission_queue_indices, + get_by_index, guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, + is_decommission_cancel_requested, load_decommission_entry_versions, local_decommission_queue_prefix, + mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts, merge_pool_status_refresh, + missing_decommission_worker_prefix, observe_decommission_terminal_reload_result, pool_meta_has_active_decommission, + reconcile_decommission_meta_buckets, require_decommission_store, reserve_decommission_start_cancelers, + resolve_decommission_bucket_done_save_result, resolve_decommission_bucket_state, + resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result, + resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result, + resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result, + resolve_decommission_partial_listing_entry, resolve_decommission_pool_meta_reload_result, + resolve_decommission_preflight_heal_result, resolve_decommission_progress_save_result, + resolve_decommission_terminal_mark_after_error_result, resolve_decommission_terminal_mark_result, + resolve_decommission_update_after_result, resolve_start_decommission_pool_meta_reload_result, + rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry, + run_decommission_listing_with_retry_and_drain, run_decommission_side_effect, should_cleanup_decommission_source_entry, should_continue_decommission_queue, should_count_decommission_version_complete, should_preserve_decommission_canceled_state, should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload, should_retry_decommission_listing, should_skip_canceled_decommission_routine, @@ -7636,6 +8183,25 @@ mod pools_tests { assert_eq!(default_decommission_bucket_concurrency(8), 4); } + #[test] + fn test_default_decommission_entry_concurrency_is_conservative() { + assert_eq!(default_decommission_entry_concurrency(0), 1); + assert_eq!(default_decommission_entry_concurrency(1), 1); + assert_eq!(default_decommission_entry_concurrency(4), 4); + assert_eq!(default_decommission_entry_concurrency(16), DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP); + } + + #[test] + fn test_decommission_entry_concurrency_clamps_operator_configuration() { + assert_eq!(clamp_decommission_entry_concurrency(0), 1); + assert_eq!(clamp_decommission_entry_concurrency(1), 1); + assert_eq!( + clamp_decommission_entry_concurrency(DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP), + DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP + ); + assert_eq!(clamp_decommission_entry_concurrency(usize::MAX), DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP); + } + #[test] fn test_split_decommission_buckets_keeps_meta_buckets_last() { let (regular, meta) = split_decommission_buckets(vec![ @@ -7843,6 +8409,190 @@ mod pools_tests { assert!(result.is_ok()); } + #[test] + fn test_decommission_entry_queue_capacity_is_bounded() { + assert_eq!(decommission_entry_queue_capacity(0), 1); + assert_eq!(decommission_entry_queue_capacity(1), 2); + assert_eq!( + decommission_entry_queue_capacity(DECOMMISSION_ENTRY_QUEUE_HARD_CAP), + DECOMMISSION_ENTRY_QUEUE_HARD_CAP + ); + assert_eq!(decommission_entry_queue_capacity(usize::MAX), DECOMMISSION_ENTRY_QUEUE_HARD_CAP); + } + + #[tokio::test] + async fn test_drain_decommission_entry_queue_waits_for_all_outstanding_entries() { + let outstanding = Arc::new(Semaphore::new(1)); + let held = outstanding + .clone() + .acquire_owned() + .await + .expect("test outstanding permit should acquire"); + let rx = CancellationToken::new(); + let drain = tokio::spawn({ + let outstanding = outstanding.clone(); + let rx = rx.clone(); + async move { drain_decommission_entry_queue(&rx, &outstanding, 1).await } + }); + + tokio::task::yield_now().await; + assert!(!drain.is_finished(), "queue drain must wait for active entry work"); + drop(held); + + let drained = tokio::time::timeout(StdDuration::from_secs(1), drain) + .await + .expect("queue drain should finish after entry completion") + .expect("queue drain task should not panic"); + assert!(!drained); + } + + #[tokio::test] + async fn test_enqueue_decommission_entry_observes_cancellation_when_queue_is_full() { + let outstanding = Arc::new(Semaphore::new(2)); + let (tx, mut queue) = tokio::sync::mpsc::channel(1); + let held = outstanding + .clone() + .acquire_owned() + .await + .expect("first queue permit should acquire"); + tx.send(QueuedDecommissionEntry { + entry: MetaCacheEntry::default(), + queue_permit: held, + }) + .await + .expect("first entry should fill the queue"); + + let rx = CancellationToken::new(); + let enqueue = tokio::spawn({ + let rx = rx.clone(); + let outstanding = outstanding.clone(); + let tx = tx.clone(); + async move { enqueue_decommission_entry(&rx, &outstanding, &tx, MetaCacheEntry::default()).await } + }); + + tokio::task::yield_now().await; + rx.cancel(); + let result = tokio::time::timeout(StdDuration::from_secs(1), enqueue) + .await + .expect("full queue enqueue should observe cancellation") + .expect("enqueue task should not panic"); + assert!(matches!(result, DecommissionEntryEnqueueResult::Canceled)); + drop(queue.recv().await); + } + + #[tokio::test] + async fn test_decommission_side_effect_gate_quiesces_before_transition() { + let operation_gate = Arc::new(tokio::sync::RwLock::new(())); + let rx = CancellationToken::new(); + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let operation = tokio::spawn({ + let operation_gate = operation_gate.clone(); + let rx = rx.clone(); + let started = started.clone(); + let release = release.clone(); + async move { + run_decommission_side_effect(&rx, &operation_gate, || async { + started.notify_one(); + release.notified().await; + Ok::<_, Error>(()) + }) + .await + } + }); + + started.notified().await; + rx.cancel(); + let transition = tokio::spawn({ + let operation_gate = operation_gate.clone(); + async move { + let _guard = operation_gate.write().await; + } + }); + tokio::task::yield_now().await; + assert!(!transition.is_finished(), "transition must wait for the in-flight side effect"); + + release.notify_one(); + let operation_result = operation.await.expect("operation task should not panic"); + assert!(matches!(operation_result, Err(Error::OperationCanceled))); + transition.await.expect("transition task should not panic"); + + let called = Arc::new(AtomicBool::new(false)); + let result = run_decommission_side_effect(&rx, &operation_gate, { + let called = called.clone(); + move || async move { + called.store(true, Ordering::SeqCst); + Ok::<_, Error>(()) + } + }) + .await; + assert!(matches!(result, Err(Error::OperationCanceled))); + assert!(!called.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn test_decommission_transition_waits_without_registered_canceler() { + let store = decommission_worker_test_store(PoolMeta::default(), vec![None]); + let operation_gate = store.ctx.decommission_operation_gate(); + let operation_guard = operation_gate.read().await; + let transition = tokio::spawn({ + let store = store.clone(); + async move { store.cancel_decommission_routines_and_wait(&[0]).await } + }); + + tokio::task::yield_now().await; + assert!( + !transition.is_finished(), + "a transition must wait for an in-flight side effect even after its canceler slot is gone" + ); + + drop(operation_guard); + tokio::time::timeout(StdDuration::from_secs(1), transition) + .await + .expect("transition should finish after the side effect") + .expect("transition task should not panic"); + } + + #[tokio::test(start_paused = true)] + async fn test_run_decommission_listing_with_retry_drains_before_each_retry() { + let attempts = Arc::new(AtomicUsize::new(0)); + let drains = Arc::new(AtomicUsize::new(0)); + let err = run_decommission_listing_with_retry_and_drain( + CancellationToken::new(), + "bucket-a".to_string(), + noop_decommission_list_callback(), + 1, + 2, + 2, + { + let attempts = attempts.clone(); + move |_| { + let attempts = attempts.clone(); + async move { + attempts.fetch_add(1, Ordering::SeqCst); + Err(Error::SlowDown) + } + } + }, + { + let drains = drains.clone(); + move || { + let drains = drains.clone(); + async move { + drains.fetch_add(1, Ordering::SeqCst); + false + } + } + }, + ) + .await + .expect_err("permanent listing failure must be returned"); + + assert!(err.to_string().contains("attempt 2/2")); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + assert_eq!(drains.load(Ordering::SeqCst), 2); + } + #[test] fn test_get_by_index_returns_value_when_in_range() { let values = vec!["a", "b", "c"]; @@ -9070,6 +9820,43 @@ mod pools_tests { assert!(!is_decommission_active(false, false, true)); } + #[test] + fn test_ensure_decommission_generation_rejects_stale_or_queued_workers() { + let generation = OffsetDateTime::UNIX_EPOCH; + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: generation, + decommission: Some(PoolDecommissionInfo { + start_time: Some(generation), + ..Default::default() + }), + }], + ..Default::default() + }; + + assert!(ensure_decommission_generation(&meta, 0, generation).is_ok()); + assert!(ensure_decommission_generation(&meta, 0, generation + Duration::seconds(1)).is_err()); + + meta.pools[0] + .decommission + .as_mut() + .expect("decommission metadata should exist") + .queued = true; + assert!(ensure_decommission_generation(&meta, 0, generation).is_err()); + + let replacement_generation = generation + Duration::seconds(2); + let info = meta.pools[0] + .decommission + .as_mut() + .expect("decommission metadata should exist"); + info.queued = false; + info.start_time = Some(replacement_generation); + assert!(ensure_decommission_generation(&meta, 0, generation).is_err()); + assert!(ensure_decommission_generation(&meta, 0, replacement_generation).is_ok()); + } + #[test] fn test_pool_meta_has_active_decommission_counts_running_and_queued_states() { let active_meta = PoolMeta { @@ -10411,7 +11198,7 @@ mod pools_tests { canceler.cancel(); let err = store - .do_decommission_in_routine(canceler.clone(), 0) + .do_decommission_in_routine(canceler.clone(), 0, Arc::new(Semaphore::new(1))) .await .expect_err("missing worker metadata should fail the routine"); @@ -10427,7 +11214,7 @@ mod pools_tests { let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(first.clone()), Some(queued.clone())]); let guards = guard_decommission_cancelers(vec![(0, first.clone()), (1, queued.clone())]); - spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards) + spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1))) .await .expect("decommission supervisor should finish after queued cleanup"); diff --git a/crates/ecstore/src/data_movement/mod.rs b/crates/ecstore/src/data_movement/mod.rs index 1f4daa703..4fdda942e 100644 --- a/crates/ecstore/src/data_movement/mod.rs +++ b/crates/ecstore/src/data_movement/mod.rs @@ -26,7 +26,7 @@ use crate::storage_api_contracts::{ namespace::NamespaceLocking as _, object::{HTTPPreconditions, ObjectOperations as _}, }; -use crate::store::ECStore; +use crate::store::{ECStore, ObjectLockDiagGuard, SourceCleanupMutationFence}; use bytes::Bytes; use rustfs_filemeta::{FileInfo, FileInfoVersions, ObjectPartInfo}; use rustfs_rio::{EtagResolvable, HashReader, HashReaderDetector, Index, TryGetIndex}; @@ -856,7 +856,6 @@ fn is_equivalent_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) fn is_superseding_unversioned_data_movement_object(source: &ObjectInfo, target: &ObjectInfo) -> bool { is_unversioned_data_movement_object(source) && is_unversioned_data_movement_object(target) - && !target.delete_marker && source .mod_time .zip(target.mod_time) @@ -1028,6 +1027,7 @@ pub(crate) enum SourceCleanupError { pub(crate) struct SourceCleanupBucketFence<'a> { pub(crate) expected_incarnation_id: Option, pub(crate) lifecycle_guard: Option<&'a rustfs_lock::NamespaceLockGuard>, + pub(crate) object_mutation_fence: Option<&'a SourceCleanupMutationFence>, } fn ensure_source_cleanup_versions_match( @@ -1065,7 +1065,9 @@ pub(crate) async fn ensure_source_cleanup_versions_unchanged( struct SourceCleanupDeleteBarrierState { bucket: String, object: String, + fence_pending: tokio::sync::Notify, arrived: tokio::sync::Notify, + is_paused: AtomicBool, release: tokio::sync::Notify, } @@ -1079,7 +1081,7 @@ pub(crate) struct SourceCleanupDeleteBarrier { } #[cfg(test)] -static SOURCE_CLEANUP_DELETE_BARRIER: std::sync::OnceLock>>> = +static SOURCE_CLEANUP_DELETE_BARRIERS: std::sync::OnceLock>>> = std::sync::OnceLock::new(); #[cfg(test)] @@ -1092,15 +1094,22 @@ impl SourceCleanupDeleteBarrier { let state = Arc::new(SourceCleanupDeleteBarrierState { bucket: bucket.to_string(), object: object.to_string(), + fence_pending: tokio::sync::Notify::new(), arrived: tokio::sync::Notify::new(), + is_paused: AtomicBool::new(false), release: tokio::sync::Notify::new(), }); - let mut slot = SOURCE_CLEANUP_DELETE_BARRIER - .get_or_init(|| std::sync::Mutex::new(None)) + let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS + .get_or_init(|| std::sync::Mutex::new(Vec::new())) .lock() .expect("source cleanup delete barrier mutex should not poison"); - assert!(slot.is_none(), "source cleanup delete barrier must be unique"); - *slot = Some(Arc::clone(&state)); + assert!( + !barriers + .iter() + .any(|barrier| barrier.bucket == bucket && barrier.object == object), + "source cleanup delete barrier must be unique per object" + ); + barriers.push(Arc::clone(&state)); Self { state } } @@ -1110,35 +1119,58 @@ impl SourceCleanupDeleteBarrier { .expect("source cleanup should reach the pre-delete barrier"); } + pub(crate) async fn wait_until_fence_pending(&self) { + tokio::time::timeout(StdDuration::from_secs(30), self.state.fence_pending.notified()) + .await + .expect("source cleanup should attempt the fixed mutation fence"); + } + + pub(crate) fn is_paused(&self) -> bool { + self.state.is_paused.load(Ordering::Acquire) + } + pub(crate) fn release(&self) { self.state.release.notify_one(); } } +#[cfg(test)] +pub(crate) fn notify_source_cleanup_mutation_fence_pending(bucket: &str, object: &str) { + let barrier = SOURCE_CLEANUP_DELETE_BARRIERS + .get_or_init(|| std::sync::Mutex::new(Vec::new())) + .lock() + .expect("source cleanup delete barrier mutex should not poison") + .iter() + .find(|barrier| barrier.bucket == bucket && barrier.object == object) + .cloned(); + if let Some(barrier) = barrier { + barrier.fence_pending.notify_one(); + } +} + #[cfg(test)] impl Drop for SourceCleanupDeleteBarrier { fn drop(&mut self) { self.state.release.notify_one(); - let mut slot = SOURCE_CLEANUP_DELETE_BARRIER - .get_or_init(|| std::sync::Mutex::new(None)) + let mut barriers = SOURCE_CLEANUP_DELETE_BARRIERS + .get_or_init(|| std::sync::Mutex::new(Vec::new())) .lock() .expect("source cleanup delete barrier mutex should not poison"); - if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { - *slot = None; - } + barriers.retain(|state| !Arc::ptr_eq(state, &self.state)); } } #[cfg(test)] async fn pause_source_cleanup_before_delete(bucket: &str, object: &str) { - let barrier = SOURCE_CLEANUP_DELETE_BARRIER - .get_or_init(|| std::sync::Mutex::new(None)) + let barrier = SOURCE_CLEANUP_DELETE_BARRIERS + .get_or_init(|| std::sync::Mutex::new(Vec::new())) .lock() .expect("source cleanup delete barrier mutex should not poison") - .as_ref() - .filter(|barrier| barrier.bucket == bucket && barrier.object == object) + .iter() + .find(|barrier| barrier.bucket == bucket && barrier.object == object) .cloned(); if let Some(barrier) = barrier { + barrier.is_paused.store(true, Ordering::Release); barrier.arrived.notify_one(); barrier.release.notified().await; } @@ -1154,11 +1186,20 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( op_label: &str, ) -> std::result::Result { let cleanup_key = encode_dir_object(object); - let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?; - let _guard = ns_lock - .get_write_lock(get_lock_acquire_timeout()) - .await - .map_err(Error::from)?; + let source_guard = if bucket_fence + .object_mutation_fence + .is_some_and(SourceCleanupMutationFence::source_lock_covered) + { + None + } else { + let ns_lock = set.new_ns_lock(bucket, cleanup_key.as_str()).await?; + Some( + ns_lock + .get_write_lock(get_lock_acquire_timeout()) + .await + .map_err(Error::from)?, + ) + }; if bucket_fence .lifecycle_guard @@ -1168,6 +1209,14 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( "{op_label}: bucket incarnation fence was lost before source cleanup" )))); } + if bucket_fence + .object_mutation_fence + .is_some_and(SourceCleanupMutationFence::is_lock_lost) + { + return Err(SourceCleanupError::Storage(Error::other(format!( + "{op_label}: object mutation fence was lost before source cleanup" + )))); + } ensure_source_cleanup_versions_unchanged(set.clone(), bucket, object, expected, allowed_missing, op_label).await?; @@ -1182,7 +1231,12 @@ pub(crate) async fn cleanup_source_entry_if_unchanged( expected_bucket_incarnation_id: bucket_fence.expected_incarnation_id, ..Default::default() }; - opts.add_namespace_lock_guard(&_guard); + if let Some(source_guard) = source_guard.as_ref() { + opts.add_namespace_lock_guard(source_guard); + } + if let Some(object_mutation_fence) = bucket_fence.object_mutation_fence { + object_mutation_fence.add_namespace_lock_fence(&mut opts); + } if let Some(bucket_lifecycle_guard) = bucket_fence.lifecycle_guard { opts.add_bucket_lifecycle_lock_guard(bucket_lifecycle_guard); } @@ -1330,6 +1384,37 @@ fn data_movement_part_upload_failure_stage(err: &Error) -> &'static str { } } +pub(crate) async fn migrate_decommission_object( + store: Arc, + pool_idx: usize, + bucket: String, + rd: GetObjectReader, + source_bucket_incarnation_id: Option, + op_label: &str, +) -> Result<()> { + let source = rd.object_info.clone(); + let _mutation_fence = store + .acquire_decommission_object_mutation_fence(&bucket, &source.name) + .await?; + let current = find_data_movement_target_info(store.as_ref(), pool_idx, &bucket, &source) + .await? + .ok_or(Error::FileNotFound)?; + if !is_equivalent_data_movement_object_identity(&source, ¤t, true, false) { + return Err(Error::FileNotFound); + } + + migrate_object_inner( + store, + pool_idx, + bucket, + rd, + source_bucket_incarnation_id, + op_label, + Some(&_mutation_fence), + ) + .await +} + pub(crate) async fn migrate_object( store: Arc, pool_idx: usize, @@ -1337,6 +1422,18 @@ pub(crate) async fn migrate_object( rd: GetObjectReader, source_bucket_incarnation_id: Option, op_label: &str, +) -> Result<()> { + migrate_object_inner(store, pool_idx, bucket, rd, source_bucket_incarnation_id, op_label, None).await +} + +async fn migrate_object_inner( + store: Arc, + pool_idx: usize, + bucket: String, + rd: GetObjectReader, + source_bucket_incarnation_id: Option, + op_label: &str, + mutation_fence: Option<&ObjectLockDiagGuard>, ) -> Result<()> { let object_info = rd.object_info.clone(); let has_part_checksums = object_info @@ -1350,7 +1447,7 @@ pub(crate) async fn migrate_object( let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx); new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; let (res, target_pool_idx, expected_bucket_incarnation_id) = match store - .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts) + .handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence) .await { Ok(res) => res, @@ -1448,7 +1545,7 @@ pub(crate) async fn migrate_object( if let Err(err) = store .clone() .complete_multipart_upload_for_data_movement( - target_pool_idx, + (target_pool_idx, mutation_fence), &bucket, &object_info.name, &res.upload_id, @@ -1609,7 +1706,7 @@ pub(crate) async fn migrate_object( let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx); put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id; let (target_pool_idx, put_result) = store - .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts) + .put_object_for_data_movement(&bucket, &object_info.name, &mut data, &put_opts, mutation_fence) .await .map_err(|err| data_movement_stage_error(op_label, "prepare_put_object", &bucket, &object_info.name, err))?; if let Err(err) = put_result { @@ -3541,25 +3638,47 @@ mod tests { } #[test] - fn test_precondition_conflict_rejects_newer_delete_marker() { - let source = ObjectInfo { - size: 128, - etag: Some("etag-source".to_string()), - mod_time: Some(OffsetDateTime::UNIX_EPOCH), - ..Default::default() - }; - let target = ObjectInfo { - delete_marker: true, - etag: None, - mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), - ..source.clone() - }; + fn test_precondition_conflict_accepts_only_newer_null_delete_marker() { + for version_id in [None, Some(Uuid::nil())] { + let source = ObjectInfo { + version_id, + size: 128, + etag: Some("etag-source".to_string()), + mod_time: Some(OffsetDateTime::UNIX_EPOCH), + ..Default::default() + }; + let target = ObjectInfo { + delete_marker: true, + etag: None, + mod_time: OffsetDateTime::UNIX_EPOCH.checked_add(time::Duration::SECOND), + ..source.clone() + }; - let should_resume = - resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(target)), &source, 0, 1) - .expect("delete marker conflict should be evaluated"); + assert!( + resolve_data_movement_overwrite_resume_result( + &Error::PreconditionFailed, + Ok(Some(target.clone())), + &source, + 0, + 1, + ) + .expect("newer null delete marker should be evaluated") + ); - assert!(!should_resume); + let mut same_time = target.clone(); + same_time.mod_time = source.mod_time; + assert!( + !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(same_time)), &source, 0, 1,) + .expect("same-generation null delete marker should be rejected") + ); + + let mut versioned = target; + versioned.version_id = Some(Uuid::new_v4()); + assert!( + !resolve_data_movement_overwrite_resume_result(&Error::PreconditionFailed, Ok(Some(versioned)), &source, 0, 1,) + .expect("a UUID delete marker must not erase a null source version") + ); + } } #[test] diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 2d7270cff..9994f8612 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -858,6 +858,7 @@ const EVENT_DISK_LOCAL_DIRECT_IO_FALLBACK: &str = "disk_local_direct_io_fallback #[cfg(target_os = "linux")] const EVENT_DISK_LOCAL_URING_LATCH_OFF: &str = "disk_local_uring_latch_off"; const EVENT_DISK_LOCAL_DELETE_FAILED: &str = "disk_local_delete_failed"; +const EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED: &str = "disk_local_delete_rollback_failed"; const EVENT_DISK_LOCAL_CHECK_PARTS: &str = "disk_local_check_parts"; const EVENT_DISK_LOCAL_ACCESS_FAILED: &str = "disk_local_access_failed"; const EVENT_DISK_LOCAL_VOLUME_SETUP_FAILED: &str = "disk_local_volume_setup_failed"; @@ -6106,6 +6107,43 @@ impl LocalDisk { Ok((bytes, modtime)) } + async fn write_missing_delete_marker( + &self, + volume: &str, + path: &str, + fi: FileInfo, + object_dir: &Path, + xl_path: &Path, + rollback_dir: Option, + ) -> Result<()> { + if let Some(rollback_dir) = rollback_dir { + let rollback_path = object_dir.join(rollback_dir.to_string()); + fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?; + fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), []) + .await + .map_err(to_file_error)?; + } + if let Err(err) = self.write_metadata("", volume, path, fi).await { + if let Some(rollback_dir) = rollback_dir + && let Err(restore_err) = restore_delete_rollback(object_dir, xl_path, rollback_dir, &self.publication_root).await + { + warn!( + event = EVENT_DISK_LOCAL_DELETE_ROLLBACK_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_DISK_LOCAL, + result = "failed", + volume, + path, + rollback_dir = %rollback_dir, + error = ?restore_err, + "Disk local delete rollback failed" + ); + } + return Err(err); + } + Ok(()) + } + async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &[FileInfo], opts: &DeleteOptions) -> Result<()> { let volume_dir = self.io_get_bucket_path(volume)?; let xlpath = self.io_get_object_path(volume, format!("{path}/{STORAGE_FORMAT_FILE}").as_str())?; @@ -6123,7 +6161,20 @@ impl LocalDisk { return restore_metadata_backup(object_dir, &xlpath, rollback_dir, &self.publication_root).await; } - let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await?; + let (data, _) = match self.read_all_data_with_dmtime(volume, volume_dir.as_path(), &xlpath).await { + Ok(data) => data, + Err(DiskError::FileNotFound) => { + // `deleted` alone can be an explicit marker purge; only + // `mark_deleted` may create metadata that was not present. + let Some(delete_marker) = fis.iter().find(|fi| fi.deleted && fi.mark_deleted).cloned() else { + return Err(DiskError::FileNotFound); + }; + return self + .write_missing_delete_marker(volume, path, delete_marker, object_dir, &xlpath, opts.old_data_dir) + .await; + } + Err(err) => return Err(err), + }; if data.is_empty() { return Err(DiskError::FileNotFound); @@ -10422,29 +10473,9 @@ impl DiskAPI for LocalDisk { } if fi.deleted && force_del_marker { - if let Some(rollback_dir) = rollback_dir { - let rollback_path = file_path.join(rollback_dir.to_string()); - fs::create_dir_all(&rollback_path).await.map_err(to_file_error)?; - fs::write(rollback_path.join(DELETE_MARKER_ROLLBACK_FILE), []) - .await - .map_err(to_file_error)?; - } - if let Err(err) = self.write_metadata("", volume, path, fi).await { - if let Some(rollback_dir) = rollback_dir - && let Err(restore_err) = - restore_delete_rollback(file_path.as_path(), &xl_path, rollback_dir, &self.publication_root).await - { - warn!( - volume, - path, - rollback_dir = %rollback_dir, - error = ?restore_err, - "failed to restore metadata after delete marker commit error" - ); - } - return Err(err); - } - return Ok(()); + return self + .write_missing_delete_marker(volume, path, fi, file_path.as_path(), &xl_path, rollback_dir) + .await; } return if fi.version_id.is_some() { diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index ecd0179e0..fb8c8de5a 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -44,6 +44,8 @@ pub const PART_TRANSACTION_ROLLBACK: &str = "rollback"; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_DISK: &str = "disk"; const EVENT_DISK_PART_ERR_UNCLASSIFIED: &str = "disk_part_err_unclassified"; +const ENV_BATCH_READ_VERSION_SERVER_PARALLELISM: &str = "RUSTFS_BATCH_READ_VERSION_SERVER_PARALLELISM"; +const BATCH_READ_VERSION_SERVER_PARALLELISM: usize = 4; pub fn part_transaction_path(part_path: &str) -> String { match part_path.rsplit_once('/') { @@ -62,6 +64,7 @@ use bytes::Bytes; use endpoint::Endpoint; use error::DiskError; use error::{Error, Result}; +use futures::stream::{self, StreamExt}; use local::LocalDisk; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; use rustfs_madmin::info_commands::DiskMetrics; @@ -417,6 +420,14 @@ impl DiskAPI for Disk { } } + #[tracing::instrument(level = "trace", skip_all)] + async fn batch_read_version(&self, req: BatchReadVersionReq) -> Result> { + match self { + Disk::Local(local_disk) => local_disk.batch_read_version(req).await, + Disk::Remote(remote_disk) => remote_disk.batch_read_version(req).await, + } + } + #[tracing::instrument(level = "trace", skip_all)] async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { match self { @@ -1028,36 +1039,47 @@ where D: DiskAPI + ?Sized, { validate_batch_read_version_item_count(req.items.len())?; + let parallelism = batch_read_version_server_parallelism(); - let mut responses = Vec::with_capacity(req.items.len()); - for (index, item) in req.items.iter().enumerate() { - let response = match disk - .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts) - .await - { - Ok(file_info) => BatchReadVersionResp { - index, - path: item.path.clone(), - version_id: item.version_id.clone(), - success: true, - file_info, - error: String::new(), - }, - Err(err) => BatchReadVersionResp { - index, - path: item.path.clone(), - version_id: item.version_id.clone(), - success: false, - file_info: FileInfo::default(), - error: err.to_string(), - }, - }; - responses.push(response); - } + let mut responses = stream::iter(req.items.into_iter().enumerate()) + .map(|(index, item)| async move { + match disk + .read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts) + .await + { + Ok(file_info) => BatchReadVersionResp { + index, + path: item.path, + version_id: item.version_id, + success: true, + file_info, + error: String::new(), + error_code: 0, + }, + Err(err) => BatchReadVersionResp { + index, + path: item.path, + version_id: item.version_id, + success: false, + file_info: FileInfo::default(), + error: err.to_string(), + error_code: err.to_u32(), + }, + } + }) + .buffer_unordered(parallelism) + .collect::>() + .await; + responses.sort_unstable_by_key(|response| response.index); Ok(responses) } +fn batch_read_version_server_parallelism() -> usize { + rustfs_utils::get_env_usize(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, BATCH_READ_VERSION_SERVER_PARALLELISM) + .clamp(1, BATCH_READ_VERSION_MAX_ITEMS) +} + #[derive(Debug, Default, Serialize, Deserialize)] pub struct CheckPartsResp { pub results: Vec, @@ -1322,6 +1344,8 @@ pub struct BatchReadVersionResp { pub success: bool, pub file_info: FileInfo, pub error: String, + #[serde(default)] + pub error_code: u32, } pub fn validate_batch_read_version_item_count(item_count: usize) -> Result<()> { @@ -1417,6 +1441,26 @@ mod tests { assert!(!partial_valid_location.valid()); } + #[test] + fn batch_read_version_server_parallelism_defaults_to_conservative_four() { + temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, None::<&str>, || { + assert_eq!(batch_read_version_server_parallelism(), 4); + }); + } + + #[test] + fn batch_read_version_server_parallelism_honors_env_with_bounds() { + temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("8"), || { + assert_eq!(batch_read_version_server_parallelism(), 8); + }); + temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("0"), || { + assert_eq!(batch_read_version_server_parallelism(), 1); + }); + temp_env::with_var(ENV_BATCH_READ_VERSION_SERVER_PARALLELISM, Some("9999"), || { + assert_eq!(batch_read_version_server_parallelism(), BATCH_READ_VERSION_MAX_ITEMS); + }); + } + /// Test FileInfoVersions find_version_index #[test] fn test_file_info_versions_find_version_index() { diff --git a/crates/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs index 2ed643ead..da4abbd39 100644 --- a/crates/ecstore/src/lib.rs +++ b/crates/ecstore/src/lib.rs @@ -81,6 +81,14 @@ pub fn shutdown_background_monitors() { cluster::rpc::shutdown_background_monitors(); } +/// Publish that the process is ready to serve user-object GET traffic. +/// +/// Experimental metadata coalescing is allowed to run only after this point so +/// startup and internal metadata reads keep the original per-disk path. +pub fn mark_get_metadata_read_version_coalescing_service_ready() { + runtime::global::mark_get_metadata_read_version_coalescing_service_ready(); +} + #[cfg(test)] mod rio_tests { #[test] diff --git a/crates/ecstore/src/object_api/types.rs b/crates/ecstore/src/object_api/types.rs index 0194667e9..fc1513584 100644 --- a/crates/ecstore/src/object_api/types.rs +++ b/crates/ecstore/src/object_api/types.rs @@ -24,7 +24,7 @@ use crate::storage_api_contracts::{ pub struct NamespaceLockFence { signals: Arc>>, #[cfg(test)] - forced_lost: Arc, + forced_lost: Arc>>, } impl Debug for NamespaceLockFence { @@ -40,13 +40,17 @@ impl NamespaceLockFence { Self { signals: Arc::default(), #[cfg(test)] - forced_lost: Arc::new(std::sync::atomic::AtomicBool::new(false)), + forced_lost: Arc::new(vec![Arc::new(std::sync::atomic::AtomicBool::new(false))]), } } pub(crate) fn is_lock_lost(&self) -> bool { #[cfg(test)] - if self.forced_lost.load(std::sync::atomic::Ordering::Acquire) { + if self + .forced_lost + .iter() + .any(|lost| lost.load(std::sync::atomic::Ordering::Acquire)) + { return true; } self.signals.iter().any(|signal| signal.is_lost()) @@ -57,27 +61,26 @@ impl NamespaceLockFence { } fn extend(&mut self, other: &Self) { - if Arc::ptr_eq(&self.signals, &other.signals) { - return; + if !Arc::ptr_eq(&self.signals, &other.signals) { + Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned()); } - Arc::make_mut(&mut self.signals).extend(other.signals.iter().cloned()); #[cfg(test)] - if other.forced_lost.load(std::sync::atomic::Ordering::Acquire) { - self.forced_lost.store(true, std::sync::atomic::Ordering::Release); + if !Arc::ptr_eq(&self.forced_lost, &other.forced_lost) { + Arc::make_mut(&mut self.forced_lost).extend(other.forced_lost.iter().cloned()); } } #[cfg(test)] pub(crate) fn lost_for_test() -> Self { let fence = Self::new(); - fence.forced_lost.store(true, std::sync::atomic::Ordering::Release); + fence.forced_lost[0].store(true, std::sync::atomic::Ordering::Release); fence } #[cfg(test)] pub(crate) fn loss_handle_for_test() -> (Self, Arc) { let fence = Self::new(); - (fence.clone(), Arc::clone(&fence.forced_lost)) + (fence.clone(), Arc::clone(&fence.forced_lost[0])) } } @@ -411,6 +414,13 @@ impl ObjectOptions { self.namespace_lock_fence.get_or_insert_with(NamespaceLockFence::new); } + #[cfg(test)] + pub(crate) fn add_namespace_lock_fence_for_test(&mut self, fence: &NamespaceLockFence) { + self.namespace_lock_fence + .get_or_insert_with(NamespaceLockFence::new) + .extend(fence); + } + pub(crate) fn ensure_lifecycle_delete_all_journal(&mut self) { self.lifecycle_delete_all_journal .get_or_insert_with(|| Arc::new(parking_lot::Mutex::new(LifecycleDeleteAllJournalState::default()))); diff --git a/crates/ecstore/src/runtime/global.rs b/crates/ecstore/src/runtime/global.rs index fcc4411f0..56ba22a0a 100644 --- a/crates/ecstore/src/runtime/global.rs +++ b/crates/ecstore/src/runtime/global.rs @@ -25,7 +25,10 @@ use lazy_static::lazy_static; use rustfs_lock::client::LockClient; use std::{ collections::HashMap, - sync::{Arc, OnceLock}, + sync::{ + Arc, OnceLock, + atomic::{AtomicBool, Ordering}, + }, time::SystemTime, }; use tokio::sync::{OnceCell, RwLock}; @@ -37,6 +40,16 @@ pub const DISK_MIN_INODES: u64 = 1000; pub const DISK_FILL_FRACTION: f64 = 0.99; pub const DISK_RESERVE_FRACTION: f64 = 0.15; +static GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY: AtomicBool = AtomicBool::new(false); + +pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() { + GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.store(true, Ordering::Release); +} + +pub(crate) fn get_metadata_read_version_coalescing_service_ready() -> bool { + GET_METADATA_READ_VERSION_COALESCING_SERVICE_READY.load(Ordering::Acquire) +} + // Global singletons for backward compatibility with MinIO port. // These should be migrated to AppContext over time. // See issue #730 for migration plan. diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 71a1898cf..9453ed02b 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -160,6 +160,10 @@ pub struct InstanceContext { /// workers (scanner/heal/tier/lifecycle) without touching another instance. /// Replaces the process-global cancel-token static. background_cancel_token: OnceLock, + /// Serializes decommission data-movement operations with cancellation and + /// a subsequent restart. Readers are held across one object side effect; + /// the transition path takes the writer after cancelling the routine. + decommission_operation_gate: Arc>, /// Resolves object-encryption material at the application boundary. object_encryption_resolver: OnceLock>, tier_delete_journal_recovery_stores: std::sync::Mutex>, @@ -200,6 +204,7 @@ impl InstanceContext { local_disk_set_drives: Arc::new(RwLock::new(Vec::new())), bucket_metadata_sys: std::sync::Mutex::new(None), background_cancel_token: OnceLock::new(), + decommission_operation_gate: Arc::new(RwLock::new(())), object_encryption_resolver: OnceLock::new(), tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()), transition_transaction_recovery_stores: std::sync::Mutex::new(HashSet::new()), @@ -218,6 +223,10 @@ impl InstanceContext { self.lock_manager.clone() } + pub(crate) fn decommission_operation_gate(&self) -> Arc> { + Arc::clone(&self.decommission_operation_gate) + } + /// Install the application-owned object-encryption resolver once. pub fn set_object_encryption_resolver( &self, diff --git a/crates/ecstore/src/services/metrics_realtime.rs b/crates/ecstore/src/services/metrics_realtime.rs index e4fc919e6..6e2e5cd7d 100644 --- a/crates/ecstore/src/services/metrics_realtime.rs +++ b/crates/ecstore/src/services/metrics_realtime.rs @@ -256,6 +256,10 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo cycle_max_duration_seconds: metrics.cycle_max_duration_seconds, cycle_max_objects: metrics.cycle_max_objects, cycle_max_directories: metrics.cycle_max_directories, + cycle_timeout_total: metrics.cycle_timeout_total, + cycle_recovery_required_total: metrics.cycle_recovery_required_total, + cycle_last_progress_age: metrics.cycle_last_progress_age, + leader_lease_without_progress: metrics.leader_lease_without_progress, bitrot_cycle_enabled: metrics.bitrot_cycle_enabled, bitrot_cycle_seconds: metrics.bitrot_cycle_seconds, scan_checkpoint: metrics.scan_checkpoint.map(|checkpoint| MadminScannerCheckpointReport { @@ -611,6 +615,10 @@ mod test { current_started: chrono_to_jiff_timestamp(current_started), last_cycle_partial_source: "usage".to_string(), last_cycle_partial_source_code: 1, + cycle_timeout_total: 3, + cycle_recovery_required_total: 2, + cycle_last_progress_age: 17, + leader_lease_without_progress: true, partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot { source: "usage".to_string(), cycles: 2, @@ -622,6 +630,10 @@ mod test { assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started)); assert_eq!(scanner.last_cycle_partial_source, "usage"); assert_eq!(scanner.last_cycle_partial_source_code, 1); + assert_eq!(scanner.cycle_timeout_total, 3); + assert_eq!(scanner.cycle_recovery_required_total, 2); + assert_eq!(scanner.cycle_last_progress_age, 17); + assert!(scanner.leader_lease_without_progress); let usage = scanner .partial_cycles_by_source .iter() diff --git a/crates/ecstore/src/services/rebalance/entry.rs b/crates/ecstore/src/services/rebalance/entry.rs index 764a68500..e9e1e2343 100644 --- a/crates/ecstore/src/services/rebalance/entry.rs +++ b/crates/ecstore/src/services/rebalance/entry.rs @@ -334,6 +334,7 @@ impl ECStore { lifecycle_guard: bucket_incarnation_fence .as_ref() .and_then(|guard| guard.namespace_lock_guard()), + ..Default::default() }, "rebalance", ), diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index d17c1c751..1a77ba267 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -53,11 +53,12 @@ use crate::diagnostics::get::{ GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure, record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled, }; -use crate::disk::disk_store::DiskStoreRenameDataExt; +use crate::disk::disk_store::{DiskStoreRenameDataExt, get_drive_metadata_timeout}; use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX; use crate::disk::{ - DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, - PartTransactionAction, STORAGE_FORMAT_FILE_BACKUP, part_transaction_path, + BATCH_READ_VERSION_MAX_ITEMS, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp, DataDirDeleteStatus, Disk, + OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK, PartTransactionAction, + STORAGE_FORMAT_FILE_BACKUP, part_transaction_path, }; use crate::erasure::coding::BitrotReader; use crate::io_support::bitrot::ShardReader; @@ -75,7 +76,7 @@ use std::{ future::Future, pin::Pin, sync::{ - OnceLock, + Arc, OnceLock, atomic::{AtomicUsize, Ordering}, }, task::{Context, Poll}, @@ -94,6 +95,242 @@ fn metadata_distribution_key(bucket: &str, object: &str) -> String { [bucket, object].join("/") } +fn read_version_coalescing_enabled() -> bool { + let enabled = || { + rustfs_utils::get_env_opt_str(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE) + .is_some_and(|value| value.eq_ignore_ascii_case("auto") || value.eq_ignore_ascii_case("on")) + }; + + #[cfg(test)] + { + enabled() + } + + #[cfg(not(test))] + { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(enabled) + } +} + +fn read_version_coalescing_delay() -> Duration { + #[cfg(test)] + { + let micros = rustfs_utils::get_env_u64( + ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + ); + Duration::from_micros(micros) + } + + #[cfg(not(test))] + { + static DELAY: OnceLock = OnceLock::new(); + *DELAY.get_or_init(|| { + Duration::from_micros(rustfs_utils::get_env_u64( + ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, + )) + }) + } +} + +struct CoalescedReadVersionRequest { + item: BatchReadVersionItem, + tx: oneshot::Sender>, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +struct ReadVersionCoalescerKey { + disk: usize, + incl_free_versions: bool, + read_data: bool, + healing: bool, +} + +impl ReadVersionCoalescerKey { + fn new(disk: &DiskStore, opts: &ReadOptions) -> Self { + Self { + disk: Arc::as_ptr(disk) as usize, + incl_free_versions: opts.incl_free_versions, + read_data: opts.read_data, + healing: opts.healing, + } + } +} + +#[derive(Default)] +struct ReadVersionCoalescer { + lanes: HashMap>, +} + +fn read_version_coalescer() -> &'static Mutex { + static COALESCER: OnceLock> = OnceLock::new(); + COALESCER.get_or_init(|| Mutex::new(ReadVersionCoalescer::default())) +} + +fn record_read_version_coalescer_event(event: &'static str, item_count: usize) { + counter!( + METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL, + "event" => event, + "item_count" => item_count.to_string() + ) + .increment(1); +} + +async fn read_version_via_coalescer( + disk: DiskStore, + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + opts: &ReadOptions, + allow_coalescing: bool, +) -> disk::error::Result { + if !allow_coalescing || !read_version_coalescing_enabled() { + return disk.read_version(org_bucket, bucket, object, version_id, opts).await; + } + if !matches!(disk.as_ref(), Disk::Remote(_)) { + record_read_version_coalescer_event("bypass_non_remote", 1); + return disk.read_version(org_bucket, bucket, object, version_id, opts).await; + } + + let (tx, rx) = oneshot::channel(); + let item = BatchReadVersionItem { + org_volume: org_bucket.to_string(), + volume: bucket.to_string(), + path: object.to_string(), + version_id: version_id.to_string(), + }; + let lane_key = ReadVersionCoalescerKey::new(&disk, opts); + let pending = { + let mut coalescer = read_version_coalescer().lock().await; + let lane = coalescer.lanes.entry(lane_key).or_default(); + let schedule_delayed_flush = lane.is_empty(); + lane.push(CoalescedReadVersionRequest { item, tx }); + if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS { + coalescer.lanes.remove(&lane_key) + } else if schedule_delayed_flush { + let disk = disk.clone(); + let task_opts = *opts; + tokio::spawn(async move { + tokio::time::sleep(read_version_coalescing_delay()).await; + flush_read_version_coalescer_lane(lane_key, disk, task_opts).await; + }); + None + } else { + None + } + }; + + if let Some(pending) = pending { + flush_read_version_coalescer_pending(lane_key, disk, *opts, pending).await; + } + + rx.await + .unwrap_or_else(|_| Err(DiskError::other("coalesced read_version response channel closed"))) +} + +async fn flush_read_version_coalescer_lane(lane_key: ReadVersionCoalescerKey, disk: DiskStore, opts: ReadOptions) { + let pending = { + let mut coalescer = read_version_coalescer().lock().await; + coalescer.lanes.remove(&lane_key).unwrap_or_default() + }; + flush_read_version_coalescer_pending(lane_key, disk, opts, pending).await; +} + +async fn flush_read_version_coalescer_pending( + lane_key: ReadVersionCoalescerKey, + disk: DiskStore, + opts: ReadOptions, + pending: Vec, +) { + if pending.is_empty() { + return; + } + + #[cfg(test)] + { + let mut observed_paths = HashSet::new(); + for request in &pending { + if observed_paths.insert(request.item.path.as_str()) { + disk_call_counters::record(&request.item.path, disk_call_counters::KIND_BATCH_READ_VERSION, lane_key.disk); + } + } + } + + let mut senders = Vec::with_capacity(pending.len()); + let mut items = Vec::with_capacity(pending.len()); + for request in pending { + senders.push(request.tx); + items.push(request.item); + } + + let expected_items = items.clone(); + record_read_version_coalescer_event("attempted_batch", items.len()); + let result = + match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts })) + .await + { + Ok(result) => result, + Err(_) => Err(DiskError::Timeout), + }; + match result { + Ok(responses) => { + let results = map_batch_read_version_responses(&expected_items, responses); + for (tx, result) in senders.into_iter().zip(results) { + let _ = tx.send(result); + } + } + Err(err) => { + let message = err.to_string(); + for tx in senders { + let _ = tx.send(Err(DiskError::other(message.clone()))); + } + } + } +} + +fn map_batch_read_version_responses( + expected_items: &[BatchReadVersionItem], + responses: Vec, +) -> Vec> { + let mut results = (0..expected_items.len()) + .map(|_| Err(DiskError::other("coalesced read_version response missing"))) + .collect::>(); + let mut seen = vec![false; expected_items.len()]; + for response in responses { + let Some(expected) = expected_items.get(response.index) else { + continue; + }; + let Some(slot) = results.get_mut(response.index) else { + continue; + }; + if seen[response.index] { + *slot = Err(DiskError::other("coalesced read_version response duplicate index")); + continue; + } + seen[response.index] = true; + if response.path != expected.path || response.version_id != expected.version_id { + *slot = Err(DiskError::other("coalesced read_version response identity mismatch")); + } else { + *slot = if response.success { + Ok(response.file_info) + } else { + Err(batch_read_version_response_error(response.error_code, response.error)) + }; + } + } + results +} + +fn batch_read_version_response_error(error_code: u32, error: String) -> DiskError { + match DiskError::from_u32(error_code) { + Some(DiskError::Io(_)) | None => DiskError::other(error), + Some(error) => error, + } +} + pub(in crate::set_disk) fn bounded_metadata_fanout_order( bucket: &str, object: &str, @@ -133,11 +370,15 @@ pub(in crate::set_disk) fn bounded_metadata_fanout_order( order } use tokio::io::{AsyncRead, ReadBuf}; -use tokio::sync::RwLock; +use tokio::sync::{Mutex, RwLock, oneshot}; use tokio::task::JoinSet; pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read"; pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP"; +const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE"; +const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS"; +const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200; +const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total"; pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE"; /// Default reader-setup strategy for the GET read path (rustfs/backlog#1215, /// #1159, #923). @@ -2356,6 +2597,7 @@ impl SetDisks { false, true, 0, + false, ) .await?; Ok((ress, errors)) @@ -2386,6 +2628,36 @@ impl SetDisks { true, caller_allows_early_stop, default_parity_count, + false, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + pub(in crate::set_disk) async fn read_all_fileinfo_observed_for_get_object( + disks: &[Option], + org_bucket: &str, + bucket: &str, + object: &str, + version_id: &str, + read_data: bool, + incl_free_versions: bool, + caller_allows_early_stop: bool, + default_parity_count: usize, + ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { + Self::read_all_fileinfo_inner( + disks, + org_bucket, + bucket, + object, + version_id, + read_data, + false, + incl_free_versions, + true, + caller_allows_early_stop, + default_parity_count, + true, ) .await } @@ -2408,6 +2680,7 @@ impl SetDisks { // subset would fail write quorum (backlog#872 regression). caller_allows_early_stop: bool, default_parity_count: usize, + allow_coalescing: bool, ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { let early_stop_enabled = caller_allows_early_stop && observe && (is_get_metadata_early_stop_enabled() || is_version_early_stop_enabled()); @@ -2424,6 +2697,7 @@ impl SetDisks { healing, incl_free_versions, default_parity_count, + allow_coalescing, ) .await; } @@ -2446,6 +2720,7 @@ impl SetDisks { healing, incl_free_versions, observe, + allow_coalescing, ) .await } @@ -2461,6 +2736,7 @@ impl SetDisks { healing: bool, incl_free_versions: bool, observe: bool, + allow_coalescing: bool, ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { let fanout_start = observe.then(Instant::now); let mut ress = Vec::with_capacity(disks.len()); @@ -2492,7 +2768,7 @@ impl SetDisks { if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) { tokio::time::sleep(delay).await; } - disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts) + read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing) .await } else { Err(DiskError::DiskNotFound) @@ -2559,6 +2835,7 @@ impl SetDisks { healing: bool, incl_free_versions: bool, default_parity_count: usize, + allow_coalescing: bool, ) -> disk::error::Result<(Vec, Vec>, MetadataFanoutDiagnostics)> { let fanout_start = Instant::now(); let mut ress = vec![FileInfo::default(); disks.len()]; @@ -2607,7 +2884,7 @@ impl SetDisks { if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) { tokio::time::sleep(delay).await; } - disk.read_version(&org_bucket, &bucket, &object, &version_id, &task_opts) + read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing) .await } else { Err(DiskError::DiskNotFound) @@ -5737,6 +6014,7 @@ pub(crate) mod disk_call_counters { /// Kind label for the per-disk `read_version` metadata RPC. pub const KIND_READ_VERSION: &str = "read_version"; + pub const KIND_BATCH_READ_VERSION: &str = "batch_read_version"; /// Registry key: (object, kind, disk_index). type CountKey = (String, String, usize); @@ -6460,6 +6738,286 @@ mod tests { drop(dirs); } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn metadata_read_version_coalescer_bypasses_local_disks() { + const DISKS: usize = 4; + let bucket = "coalesced-read-version-local-bypass-bucket"; + let object_a = "coalesced-local-object-a"; + let object_b = "coalesced-local-object-b"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_metadata_fanout_fileinfo(&disks, bucket, object_a, None).await; + install_metadata_fanout_fileinfo(&disks, bucket, object_b, None).await; + + temp_env::async_with_vars( + [ + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto")), + (ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS, Some("5000")), + ], + async { + let calls = disk_call_counters::observe(object_a); + let disks_a = disks.clone(); + let disks_b = disks.clone(); + let read_a = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed_for_get_object( + &disks_a, "", bucket, object_a, "", false, false, false, 2, + ) + .await + .map(|(file_infos, errors, _)| (file_infos, errors)) + }); + tokio::task::yield_now().await; + let read_b = tokio::spawn(async move { + SetDisks::read_all_fileinfo_observed_for_get_object( + &disks_b, "", bucket, object_b, "", false, false, false, 2, + ) + .await + .map(|(file_infos, errors, _)| (file_infos, errors)) + }); + + let (metadata_a, errs_a) = read_a + .await + .expect("first read task should not panic") + .expect("first coalesced read should resolve"); + let (metadata_b, errs_b) = read_b + .await + .expect("second read task should not panic") + .expect("second coalesced read should resolve"); + + assert_eq!(metadata_a.iter().filter(|fi| fi.name == object_a).count(), DISKS); + assert_eq!(metadata_b.iter().filter(|fi| fi.name == object_b).count(), DISKS); + assert!(errs_a.iter().all(Option::is_none)); + assert!(errs_b.iter().all(Option::is_none)); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + DISKS as u64, + "local disks still execute the ordinary per-disk read_version path" + ); + assert_eq!( + calls.total(disk_call_counters::KIND_BATCH_READ_VERSION), + 0, + "GET coalescing targets internode RPC count only and must not batch local disk reads" + ); + }, + ) + .await; + + drop(dirs); + } + + #[tokio::test] + async fn metadata_read_version_coalescer_requires_get_object_intent() { + const DISKS: usize = 4; + let bucket = "coalesced-read-version-default-bypass-bucket"; + let object = "default-bypass-object"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_metadata_fanout_fileinfo(&disks, bucket, object, None).await; + + temp_env::async_with_vars([(ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE, Some("auto"))], async { + let calls = disk_call_counters::observe(object); + let (metadata, errs) = SetDisks::read_all_fileinfo(&disks, "", bucket, object, "", false, false, false) + .await + .expect("default metadata read should resolve"); + + assert_eq!(metadata.iter().filter(|fi| fi.name == object).count(), DISKS); + assert!(errs.iter().all(Option::is_none)); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), DISKS as u64); + assert_eq!( + calls.total(disk_call_counters::KIND_BATCH_READ_VERSION), + 0, + "non-GET metadata paths must bypass coalescer even when the env gate is enabled" + ); + }) + .await; + + drop(dirs); + } + + #[test] + fn batch_read_version_response_mapping_preserves_index_and_errors() { + let expected_items = vec![ + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-a".to_string(), + version_id: "v-a".to_string(), + }, + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-b".to_string(), + version_id: "v-b".to_string(), + }, + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-c".to_string(), + version_id: "v-c".to_string(), + }, + ]; + let ok_file_info = FileInfo { + name: "object-a".to_string(), + ..Default::default() + }; + let responses = vec![ + BatchReadVersionResp { + index: 2, + path: "object-c".to_string(), + version_id: "v-c".to_string(), + success: false, + file_info: FileInfo::default(), + error: "disk read failed".to_string(), + error_code: 0, + }, + BatchReadVersionResp { + index: 0, + path: "object-a".to_string(), + version_id: "v-a".to_string(), + success: true, + file_info: ok_file_info, + error: String::new(), + error_code: 0, + }, + ]; + + let mut results = map_batch_read_version_responses(&expected_items, responses).into_iter(); + let first = results + .next() + .expect("slot 0 should exist") + .expect("slot 0 should map the success response by index"); + assert_eq!(first.name, "object-a"); + + let missing = results + .next() + .expect("slot 1 should exist") + .expect_err("slot 1 should stay missing"); + assert!( + missing.to_string().contains("response missing"), + "unexpected missing response error: {missing}" + ); + + let failed = results + .next() + .expect("slot 2 should exist") + .expect_err("slot 2 should map the response error"); + assert!(failed.to_string().contains("disk read failed"), "unexpected per-item error: {failed}"); + assert!(results.next().is_none()); + } + + #[test] + fn batch_read_version_response_mapping_preserves_typed_not_found_errors() { + let expected_items = vec![ + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-a".to_string(), + version_id: "v-a".to_string(), + }, + BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-b".to_string(), + version_id: "v-b".to_string(), + }, + ]; + let results = map_batch_read_version_responses( + &expected_items, + vec![ + BatchReadVersionResp { + index: 0, + path: "object-a".to_string(), + version_id: "v-a".to_string(), + success: false, + file_info: FileInfo::default(), + error: DiskError::FileNotFound.to_string(), + error_code: DiskError::FileNotFound.to_u32(), + }, + BatchReadVersionResp { + index: 1, + path: "object-b".to_string(), + version_id: "v-b".to_string(), + success: false, + file_info: FileInfo::default(), + error: DiskError::FileVersionNotFound.to_string(), + error_code: DiskError::FileVersionNotFound.to_u32(), + }, + ], + ); + + assert!(matches!(results.first().expect("slot 0 should exist"), Err(DiskError::FileNotFound))); + assert!(matches!( + results.get(1).expect("slot 1 should exist"), + Err(DiskError::FileVersionNotFound) + )); + } + + #[test] + fn batch_read_version_response_mapping_rejects_identity_mismatch_and_duplicate_index() { + let expected_items = vec![BatchReadVersionItem { + org_volume: String::new(), + volume: "bucket".to_string(), + path: "object-a".to_string(), + version_id: "v-a".to_string(), + }]; + let mismatched = map_batch_read_version_responses( + &expected_items, + vec![BatchReadVersionResp { + index: 0, + path: "object-b".to_string(), + version_id: "v-a".to_string(), + success: true, + file_info: FileInfo { + name: "object-b".to_string(), + ..Default::default() + }, + error: String::new(), + error_code: 0, + }], + ) + .pop() + .expect("slot 0 should exist") + .expect_err("identity mismatch should fail closed"); + assert!( + mismatched.to_string().contains("identity mismatch"), + "unexpected mismatch error: {mismatched}" + ); + + let duplicate = map_batch_read_version_responses( + &expected_items, + vec![ + BatchReadVersionResp { + index: 0, + path: "object-a".to_string(), + version_id: "v-a".to_string(), + success: true, + file_info: FileInfo { + name: "object-a".to_string(), + ..Default::default() + }, + error: String::new(), + error_code: 0, + }, + BatchReadVersionResp { + index: 0, + path: "object-a".to_string(), + version_id: "v-a".to_string(), + success: true, + file_info: FileInfo { + name: "object-a".to_string(), + ..Default::default() + }, + error: String::new(), + error_code: 0, + }, + ], + ) + .pop() + .expect("slot 0 should exist") + .expect_err("duplicate response index should fail closed"); + assert!( + duplicate.to_string().contains("duplicate index"), + "unexpected duplicate error: {duplicate}" + ); + } + /// Isolation guard: unobserved objects record nothing (so parallel tests do /// not inflate one another), and a scope clears its own counts on drop. #[tokio::test] diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index e64e1704f..0422ba94e 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -735,8 +735,12 @@ pub(crate) use core::io_primitives::disk_call_counters; mod ctx; mod metadata; mod ops; +#[cfg(test)] +pub(crate) use ops::multipart::NewMultipartUploadCommitObservation; #[cfg(any(test, feature = "test-util"))] pub use ops::multipart::{MultipartCommitBarrier, MultipartCommitPause}; +#[cfg(test)] +pub(crate) use ops::object::DeleteObjectCommitBarrier; #[cfg(feature = "test-util")] pub(crate) use ops::object::TransitionCleanupStoreBarrier as SetDiskTransitionCleanupStoreBarrier; pub(crate) use ops::object::body_cache_plaintext_len; @@ -3025,6 +3029,16 @@ pub struct SetDisks { storage_class_config_override: Arc>>>, } +// DistributedLock sends the raw ObjectKey to its clients; LockRegistry clones +// each endpoint's canonical Arc, so an exact Arc set identifies the lock domain. +pub(crate) fn same_distributed_lock_domain(left: &[Arc], right: &[Arc]) -> bool { + left.iter() + .all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client))) + && right + .iter() + .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client))) +} + const ERASURE_CACHE_MAX_ENTRIES: usize = 32; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -3600,6 +3614,15 @@ impl SetDisks { &self.ctx } + /// Whether both sets' namespace-lock implementations cover the same object key. + pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool { + match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) { + (false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager), + (true, true) => same_distributed_lock_domain(&self.lockers, &other.lockers), + _ => false, + } + } + /// The lock manager this set actually uses (test-only; Phase 5 Slice 3). #[cfg(test)] pub(crate) fn local_lock_manager_for_test(&self) -> &Arc { @@ -4584,11 +4607,11 @@ fn should_preserve_delete_replication_state(opts: &ObjectOptions) -> bool { } fn should_force_delete_marker_for_missing_version(opts: &ObjectOptions) -> bool { - opts.delete_marker || (opts.versioned && opts.version_id.is_none() && !opts.data_movement) + opts.delete_marker || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.data_movement) } fn resolve_delete_version_state(opts: &ObjectOptions, goi: &ObjectInfo, version_found: bool) -> (bool, bool) { - let mut mark_delete = goi.version_id.is_some() || (opts.versioned && opts.version_id.is_none()); + let mut mark_delete = goi.version_id.is_some() || ((opts.versioned || opts.version_suspended) && opts.version_id.is_none()); let mut delete_marker = opts.versioned; if opts.version_id.is_some() { diff --git a/crates/ecstore/src/set_disk/ops/multipart.rs b/crates/ecstore/src/set_disk/ops/multipart.rs index 6af71db17..5aa59b158 100644 --- a/crates/ecstore/src/set_disk/ops/multipart.rs +++ b/crates/ecstore/src/set_disk/ops/multipart.rs @@ -32,6 +32,8 @@ use crate::crash_inject::{self, CrashPoint}; use crate::multipart_listing::paginate_multipart_listing; use futures::{StreamExt, stream}; use std::future::Future; +#[cfg(test)] +use std::sync::atomic::AtomicBool; #[cfg(any(test, feature = "test-util"))] use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; @@ -65,6 +67,7 @@ impl StaleMultipartCleanupGuard { #[cfg(any(test, feature = "test-util"))] #[derive(Clone, Copy, PartialEq, Eq)] pub enum MultipartCommitPause { + NewUploadBeforeLockLost, PutPartBeforeLockAcquire, PutPartBeforeLockLost, PutPartAfterRename, @@ -156,6 +159,72 @@ impl Drop for MultipartCommitBarrier { } } +#[cfg(test)] +struct NewMultipartUploadCommitObservationState { + bucket: String, + object: String, + committed: AtomicBool, +} + +#[cfg(test)] +pub(crate) struct NewMultipartUploadCommitObservation { + state: Arc, +} + +#[cfg(test)] +static NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl NewMultipartUploadCommitObservation { + pub(crate) fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(NewMultipartUploadCommitObservationState { + bucket: bucket.to_string(), + object: object.to_string(), + committed: AtomicBool::new(false), + }); + let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("new multipart upload commit observation mutex should not poison"); + assert!(slot.is_none(), "new multipart upload commit observation must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) fn committed(&self) -> bool { + self.state.committed.load(Ordering::Acquire) + } +} + +#[cfg(test)] +impl Drop for NewMultipartUploadCommitObservation { + fn drop(&mut self) { + let mut slot = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("new multipart upload commit observation mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +fn observe_new_multipart_upload_commit(bucket: &str, object: &str) { + let state = NEW_MULTIPART_UPLOAD_COMMIT_OBSERVATION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("new multipart upload commit observation mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket && state.object == object) + .cloned(); + if let Some(state) = state { + state.committed.store(true, Ordering::Release); + } +} + #[cfg(any(test, feature = "test-util"))] async fn pause_multipart_commit(bucket: &str, object: &str, pause: MultipartCommitPause) { let barrier = { @@ -1615,6 +1684,30 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_uuid.as_str(), opts.data_movement); + #[cfg(any(test, feature = "test-util"))] + pause_multipart_commit(bucket, object, MultipartCommitPause::NewUploadBeforeLockLost).await; + if _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "new_multipart_upload_commit", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } + if opts + .namespace_lock_fence + .as_ref() + .is_some_and(NamespaceLockFence::is_lock_lost) + { + return Err(StorageError::NamespaceLockQuorumUnavailable { + mode: "new_multipart_upload_outer_lock", + bucket: bucket.to_string(), + object: object.to_string(), + required: 1, + achieved: 0, + }); + } ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?; Self::write_unique_file_info( &shuffle_disks, @@ -1626,6 +1719,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks { ) .await .map_err(|e| to_object_err(e.into(), vec![bucket, object]))?; + #[cfg(test)] + observe_new_multipart_upload_commit(bucket, object); // evalDisks diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index bed7a8f18..02a997883 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -1294,7 +1294,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { (prepared.snapshot, prepared.object_info) } else { match self - .get_object_fileinfo( + .get_object_fileinfo_for_get_object_reader( bucket, object, opts, @@ -2497,6 +2497,7 @@ impl SetDisks { }) .await?, ); + notify_put_object_commit_namespace_acquired(bucket, object); } #[cfg(not(any(test, feature = "test-util")))] { @@ -4649,6 +4650,7 @@ struct PutObjectCommitBarrierState { arrived: tokio::sync::Notify, release: tokio::sync::Notify, namespace_pending: tokio::sync::Notify, + namespace_acquired: std::sync::atomic::AtomicBool, } #[cfg(any(test, feature = "test-util"))] @@ -4670,6 +4672,7 @@ impl PutObjectCommitBarrier { arrived: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), namespace_pending: tokio::sync::Notify::new(), + namespace_acquired: std::sync::atomic::AtomicBool::new(false), }); let mut slot = PUT_OBJECT_COMMIT_BARRIER .get_or_init(|| std::sync::Mutex::new(Vec::new())) @@ -4704,6 +4707,10 @@ impl PutObjectCommitBarrier { .await .expect("put object should wait for the namespace lock after leaving the commit barrier"); } + + pub fn namespace_acquired(&self) -> bool { + self.state.namespace_acquired.load(std::sync::atomic::Ordering::Acquire) + } } #[cfg(any(test, feature = "test-util"))] @@ -4760,6 +4767,22 @@ fn notify_put_object_commit_namespace_pending(bucket: &str, object: &str) { } } +#[cfg(any(test, feature = "test-util"))] +fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) { + let barrier = PUT_OBJECT_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(Vec::new())) + .lock() + .expect("put object commit barrier mutex should not poison") + .iter() + .find(|barrier| { + barrier.bucket == bucket && barrier.object == object && barrier.pause == PutObjectCommitPause::BeforeNamespace + }) + .cloned(); + if let Some(barrier) = barrier { + barrier.namespace_acquired.store(true, std::sync::atomic::Ordering::Release); + } +} + #[cfg(test)] struct DeleteObjectCommitBarrierState { bucket: String, @@ -4769,7 +4792,7 @@ struct DeleteObjectCommitBarrierState { } #[cfg(test)] -struct DeleteObjectCommitBarrier { +pub(crate) struct DeleteObjectCommitBarrier { state: Arc, } @@ -4779,7 +4802,7 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock Self { + pub(crate) fn install(bucket: &str, object: &str) -> Self { let state = Arc::new(DeleteObjectCommitBarrierState { bucket: bucket.to_string(), object: object.to_string(), @@ -4795,13 +4818,13 @@ impl DeleteObjectCommitBarrier { Self { state } } - async fn wait_until_paused(&self) { + pub(crate) async fn wait_until_paused(&self) { tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) .await .expect("delete object should reach the deterministic commit barrier"); } - fn release(&self) { + pub(crate) fn release(&self) { self.state.release.notify_one(); } } @@ -5923,6 +5946,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { if dobj.version_id.is_none() && (version_suspended || versioned) { vr.mod_time = Some(OffsetDateTime::now_utc()); vr.deleted = true; + vr.mark_deleted = true; if versioned { vr.version_id = Some(Uuid::new_v4()); } @@ -11809,6 +11833,7 @@ mod transition_upload_integrity_tests { crate::data_movement::SourceCleanupBucketFence { expected_incarnation_id: None, lifecycle_guard: Some(&bucket_guard), + ..Default::default() }, "test_data_movement", ) diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 2b556b143..15e7e4e2a 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -259,10 +259,33 @@ impl SetDisks { read_data: bool, caller_allows_early_stop: bool, ) -> Result { - self.get_object_fileinfo_gated(bucket, object, opts, read_data, caller_allows_early_stop) + self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, caller_allows_early_stop, false) .await } + #[tracing::instrument(level = "debug", skip(self))] + #[hotpath::measure(impl_type = "SetDisks")] + pub(super) async fn get_object_fileinfo_for_get_object_reader( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + read_data: bool, + caller_allows_early_stop: bool, + ) -> Result { + let allow_read_version_coalescing = !crate::bucket::utils::is_meta_bucketname(bucket) + && crate::runtime::global::get_metadata_read_version_coalescing_service_ready(); + self.get_object_fileinfo_gated_inner( + bucket, + object, + opts, + read_data, + caller_allows_early_stop, + allow_read_version_coalescing, + ) + .await + } + /// Like `get_object_fileinfo`, but `allow_early_stop=false` forces the full /// quorum fanout. Read-before-write callers (object tagging) must use this: /// the returned online-disk set is the write target, and the early-stop @@ -275,6 +298,20 @@ impl SetDisks { opts: &ObjectOptions, read_data: bool, allow_early_stop: bool, + ) -> Result { + self.get_object_fileinfo_gated_inner(bucket, object, opts, read_data, allow_early_stop, false) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn get_object_fileinfo_gated_inner( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + read_data: bool, + allow_early_stop: bool, + allow_read_version_coalescing: bool, ) -> Result { let vid = opts.version_id.clone().unwrap_or_default(); let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); @@ -337,19 +374,34 @@ impl SetDisks { // read_all_fileinfo_observed (see read_all_fileinfo_early_stop in // core/io_primitives.rs); unsafe requests and callers that opt out // (allow_early_stop=false) fall back to full-wait. - let (mut parts_metadata, errs, metadata_fanout_diagnostics) = Self::read_all_fileinfo_observed( - &disks, - "", - bucket, - object, - vid.as_str(), - read_data, - false, - opts.incl_free_versions, - allow_early_stop, - self.default_parity_count, - ) - .await?; + let (mut parts_metadata, errs, metadata_fanout_diagnostics) = if allow_read_version_coalescing { + Self::read_all_fileinfo_observed_for_get_object( + &disks, + "", + bucket, + object, + vid.as_str(), + read_data, + opts.incl_free_versions, + allow_early_stop, + self.default_parity_count, + ) + .await? + } else { + Self::read_all_fileinfo_observed( + &disks, + "", + bucket, + object, + vid.as_str(), + read_data, + false, + opts.incl_free_versions, + allow_early_stop, + self.default_parity_count, + ) + .await? + }; let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) { GET_OBJECT_PATH_INTERNAL_META } else { diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index d8a029f37..32a7a23fe 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -616,7 +616,7 @@ mod tests { storage_api_contracts::{ bucket::{BucketOperations as _, MakeBucketOptions}, multipart::MultipartOperations as _, - object::{ObjectIO, ObjectOperations as _}, + object::{ObjectIO, ObjectOperations as _, ObjectToDelete}, range::HTTPRangeSpec, }, }; @@ -624,9 +624,9 @@ mod tests { use futures::{StreamExt as _, TryStreamExt as _}; use http::HeaderMap; use rustfs_config::server_config::KVS; - use rustfs_filemeta::ObjectPartInfo; #[cfg(feature = "test-util")] use rustfs_filemeta::{FileInfo, FileMeta}; + use rustfs_filemeta::{FileInfoVersions, MetaCacheEntry, ObjectPartInfo}; #[cfg(feature = "test-util")] use rustfs_protos::{TIER_MUTATION_RPC_PROTOCOL_VERSION, TierMutationRpcPhase}; use rustfs_rio::{Checksum, ChecksumType}; @@ -1240,6 +1240,212 @@ mod tests { shutdown.cancel(); } + async fn migrate_versioned_decommission_test_object( + store: &Arc, + bucket: &str, + object: &str, + payload: &[u8], + op_label: &'static str, + ) -> (uuid::Uuid, FileInfoVersions) { + let mut source = PutObjReader::from_vec(payload.to_vec()); + let source_info = store.pools[0] + .put_object( + bucket, + object, + &mut source, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect("write versioned source to the pool being decommissioned"); + let source_version = source_info.version_id.expect("versioned source must have a version ID"); + let expected_source_versions = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("source versions should be readable before migration") + .expect("source versions should exist before migration"); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + bucket, + object, + crate::set_disk::PutObjectCommitPause::AfterNamespace, + ); + let migration_store = Arc::clone(store); + let migration_bucket = bucket.to_string(); + let migration_object = object.to_string(); + let migration = tokio::spawn(async move { + let source_reader = migration_store.pools[0] + .get_object_reader( + &migration_bucket, + &migration_object, + None, + HeaderMap::new(), + &ObjectOptions { + versioned: true, + version_id: Some(source_version.to_string()), + no_lock: true, + data_movement: true, + raw_data_movement_read: true, + ..Default::default() + }, + ) + .await?; + crate::data_movement::migrate_decommission_object(migration_store, 0, migration_bucket, source_reader, None, op_label) + .await + }); + barrier.wait_until_paused().await; + barrier.release(); + migration + .await + .expect("versioned decommission migration task should join") + .expect("versioned decommission migration should commit"); + + (source_version, expected_source_versions) + } + + async fn mark_test_pool_decommissioning(store: &Arc, pool_idx: usize) { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + + async fn write_decommission_test_multipart_source( + store: &Arc, + pool_idx: usize, + bucket: &str, + object: &str, + ) { + let pool = &store.pools[pool_idx]; + let upload = pool + .new_multipart_upload(bucket, object, &ObjectOptions::default()) + .await + .expect("create decommission multipart source upload"); + let first_part = vec![b'm'; 5 * 1024 * 1024]; + let second_part = b"decommission multipart tail".to_vec(); + let mut completed_parts = Vec::with_capacity(2); + for (part_number, body) in [(1, first_part), (2, second_part)] { + let mut reader = PutObjReader::from_vec(body); + let part = pool + .put_object_part(bucket, object, &upload.upload_id, part_number, &mut reader, &ObjectOptions::default()) + .await + .expect("write decommission multipart source part"); + completed_parts.push(crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }); + } + pool.clone() + .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &ObjectOptions::default()) + .await + .expect("complete decommission multipart source object"); + } + + async fn assert_pool_object_present(pool: &Arc, bucket: &str, object: &str) { + pool.get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect("expected object generation must remain present"); + } + + async fn assert_pool_object_absent(pool: &Arc, bucket: &str, object: &str) { + let err = pool + .get_object_info(bucket, object, &ObjectOptions::default()) + .await + .expect_err("fenced decommission target must remain absent"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _) | StorageError::VersionNotFound(_, _, _)), + "unexpected fenced target result: {err:?}" + ); + } + + async fn write_suspended_decommission_source(store: &Arc, bucket: &str, object: &str) { + let mut reader = PutObjReader::from_vec(b"suspended source generation".to_vec()); + let source = store.pools[0] + .put_object( + bucket, + object, + &mut reader, + &ObjectOptions { + version_suspended: true, + mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND), + ..Default::default() + }, + ) + .await + .expect("write suspended null source version"); + assert!( + source.version_id.is_none_or(|version_id| version_id.is_nil()), + "suspended source must use the null version identity" + ); + } + + async fn assert_suspended_null_source_present(store: &Arc, bucket: &str, object: &str) { + let versions = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("suspended source versions should be readable") + .expect("suspended source must exist before worker convergence"); + assert!( + versions + .versions + .iter() + .any(|version| !version.deleted && version.version_id.is_none_or(|version_id| version_id.is_nil())), + "the source pool must retain its null data version while DELETE owns the fixed fence" + ); + } + + async fn assert_suspended_decommission_converged(store: &Arc, bucket: &str, object: &str) { + let source_versions = store.pools[0] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("source versions should remain readable after suspended convergence"); + assert!( + source_versions.is_none_or(|versions| versions.versions.is_empty()), + "worker convergence must remove only the decommissioned source null version" + ); + + let target_versions = store.pools[1] + .get_disks_by_key(object) + .load_file_info_versions_exact(bucket, object) + .await + .expect("active target versions should be readable") + .expect("active target must retain the suspended DELETE marker"); + assert!( + matches!(target_versions.versions.as_slice(), [marker] if marker.deleted && marker.version_id.is_none_or(|version_id| version_id.is_nil())), + "active target must contain only its null delete marker: {target_versions:?}" + ); + + let err = store + .get_object_info( + bucket, + object, + &ObjectOptions { + version_suspended: true, + ..Default::default() + }, + ) + .await + .expect_err("the active null delete marker must hide the migrated source generation"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _)), + "unexpected suspended latest-object result: {err:?}" + ); + } + #[tokio::test] #[serial_test::serial(storage_class_env)] async fn tag_updates_skip_active_rebalance_source_pool() { @@ -2766,6 +2972,1150 @@ mod tests { .expect_err("suspended delete must remove the requested UUID version"); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn decommission_entry_carries_migration_and_cleanup_mutation_fences() { + let temp_dir = tempfile::tempdir().expect("create decommission delete-fence store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "decommission-delete-fence", + &[(2, 4), (1, 4)], + CancellationToken::new(), + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-delete-fence-{}", uuid::Uuid::new_v4()); + let object = (0..128) + .map(|index| format!("object-{index}.bin")) + .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1) + .expect("the deterministic object search should select source set 1"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create decommission delete-fence bucket"); + let mut source = PutObjReader::from_vec(b"source generation".to_vec()); + store.pools[0] + .put_object(&bucket, &object, &mut source, &ObjectOptions::default()) + .await + .expect("write source object to the pool being decommissioned"); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + assert!(store.is_suspended(0).await, "pool 0 must be a suspended decommission source"); + + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + &object, + crate::set_disk::PutObjectCommitPause::AfterNamespace, + ); + let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object); + let source_set = store.pools[0].get_disks_by_key(&object); + assert_eq!(source_set.set_index, 1, "the source entry must exercise the non-fixed set cleanup lock"); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker_object = object.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: worker_object, + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + barrier.wait_until_paused().await; + + let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete_object = object.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object(&delete_bucket, &delete_object, ObjectOptions::default()) + .await + }); + delete_barrier.wait_until_paused().await; + delete_barrier.release_and_wait_until_namespace_pending().await; + assert!( + !delete_barrier.namespace_acquired() && !delete.is_finished(), + "DELETE must remain before namespace acquisition behind the decommission worker's target-commit mutation fence" + ); + + barrier.release(); + cleanup_barrier.wait_until_paused().await; + drop(barrier); + + let fixed_set = Arc::clone(&store.pools[0].disk_set[0]); + let fixed_mutation_barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + &object, + crate::set_disk::PutObjectCommitPause::BeforeNamespace, + ); + let mutation_bucket = bucket.clone(); + let mutation_object = object.clone(); + let fixed_mutation = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(b"fixed-domain replacement".to_vec()); + fixed_set + .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default()) + .await + }); + fixed_mutation_barrier.wait_until_paused().await; + fixed_mutation_barrier.release_and_wait_until_namespace_pending().await; + assert!( + !fixed_mutation_barrier.namespace_acquired() && !fixed_mutation.is_finished(), + "the source cleanup must retain the fixed mutation fence before the set-0 mutation acquires its namespace" + ); + fixed_mutation.abort(); + assert!( + fixed_mutation + .await + .expect_err("the fixed-domain mutation should be canceled") + .is_cancelled(), + "the competing fixed-domain mutation must remain cancelable while blocked" + ); + drop(fixed_mutation_barrier); + + cleanup_barrier.release(); + worker + .await + .expect("decommission entry worker should join") + .expect("decommission entry should migrate and clean its source"); + delete + .await + .expect("DELETE task should join") + .expect("DELETE should remove the source and migrated target generations"); + + for pool in &store.pools { + let err = pool + .get_object_info(&bucket, &object, &ObjectOptions::default()) + .await + .expect_err("DELETE must remove the source and migrated target copies"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _)), + "unexpected post-delete pool result: {err:?}" + ); + } + let err = store + .get_object_info(&bucket, &object, &ObjectOptions::default()) + .await + .expect_err("the deleted generation must not become visible again"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _)), + "unexpected post-delete store result: {err:?}" + ); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn decommission_outer_fence_loss_blocks_target_put_commit() { + let temp_dir = tempfile::tempdir().expect("create decommission PUT fence-loss store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-put-fence-loss", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-put-fence-loss-{}", uuid::Uuid::new_v4()); + let object = "ordinary.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create decommission PUT fence-loss bucket"); + let mut source = PutObjReader::from_vec(b"source generation".to_vec()); + store.pools[0] + .put_object(&bucket, object, &mut source, &ObjectOptions::default()) + .await + .expect("write decommission PUT source"); + mark_test_pool_decommissioning(&store, 0).await; + + let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install( + &bucket, + object, + crate::store::object::DecommissionMutationFenceTestPhase::Migration, + ); + let barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + object, + crate::set_disk::PutObjectCommitPause::BeforeQuotaRename, + ); + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + barrier.wait_until_paused().await; + loss_hook.mark_lost(); + barrier.release(); + drop(barrier); + worker + .await + .expect("decommission PUT fence-loss worker should join") + .expect("a fenced migration failure should remain retryable at entry scope"); + + assert_pool_object_absent(&store.pools[1], &bucket, object).await; + assert_pool_object_present(&store.pools[0], &bucket, object).await; + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn decommission_outer_fence_loss_blocks_multipart_commits() { + let temp_dir = tempfile::tempdir().expect("create decommission multipart fence-loss store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-multipart-fence-loss", &[4, 4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-mpu-fence-loss-{}", uuid::Uuid::new_v4()); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create decommission multipart fence-loss bucket"); + for object in ["new-upload.bin", "complete.bin"] { + write_decommission_test_multipart_source(&store, 0, &bucket, object).await; + } + mark_test_pool_decommissioning(&store, 0).await; + + for (object, pause) in [ + ("new-upload.bin", crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost), + ("complete.bin", crate::set_disk::MultipartCommitPause::BeforeLockLost), + ] { + let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install( + &bucket, + object, + crate::store::object::DecommissionMutationFenceTestPhase::Migration, + ); + let commit_observation = (pause == crate::set_disk::MultipartCommitPause::NewUploadBeforeLockLost) + .then(|| crate::set_disk::NewMultipartUploadCommitObservation::install(&bucket, object)); + let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause); + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + barrier.wait_until_paused().await; + loss_hook.mark_lost(); + barrier.release(); + drop(barrier); + worker + .await + .expect("decommission multipart fence-loss worker should join") + .expect("a fenced multipart migration failure should remain retryable at entry scope"); + + if let Some(commit_observation) = commit_observation { + assert!( + !commit_observation.committed(), + "new multipart upload metadata must not commit after the outer fence is lost" + ); + } + assert_pool_object_absent(&store.pools[1], &bucket, object).await; + assert_pool_object_present(&store.pools[0], &bucket, object).await; + let uploads = store.pools[1] + .list_multipart_uploads(&bucket, object, None, None, None, 100) + .await + .expect("list target multipart uploads after fenced migration"); + assert!(uploads.uploads.is_empty(), "fenced multipart migration must not retain target staging"); + } + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn decommission_outer_fence_loss_blocks_source_cleanup_delete_commit() { + let temp_dir = tempfile::tempdir().expect("create decommission cleanup fence-loss store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "decommission-cleanup-fence-loss", &[4, 4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-cleanup-fence-loss-{}", uuid::Uuid::new_v4()); + let object = "cleanup.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create decommission cleanup fence-loss bucket"); + let mut source = PutObjReader::from_vec(b"source generation".to_vec()); + store.pools[0] + .put_object(&bucket, object, &mut source, &ObjectOptions::default()) + .await + .expect("write decommission cleanup source"); + mark_test_pool_decommissioning(&store, 0).await; + + let loss_hook = crate::store::object::DecommissionMutationFenceLossHook::install( + &bucket, + object, + crate::store::object::DecommissionMutationFenceTestPhase::SourceCleanup, + ); + let barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object); + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + barrier.wait_until_paused().await; + loss_hook.mark_lost(); + barrier.release(); + drop(barrier); + let err = worker + .await + .expect("decommission cleanup fence-loss worker should join") + .expect_err("source cleanup must fail after its outer fence is lost"); + assert!( + err.to_string().contains("delete_object_commit"), + "cleanup failure must come from the delete commit fence: {err:?}" + ); + + assert_pool_object_present(&store.pools[0], &bucket, object).await; + assert_pool_object_present(&store.pools[1], &bucket, object).await; + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn reverse_decommission_reuses_fixed_target_fence_for_put_and_multipart() { + let temp_dir = tempfile::tempdir().expect("create reverse decommission store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "reverse-decommission-fixed-target", + &[(1, 4), (1, 4)], + CancellationToken::new(), + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("reverse-decom-fixed-target-{}", uuid::Uuid::new_v4()); + let object = "ordinary.bin"; + let object_body = b"reverse ordinary generation".to_vec(); + let multipart_object = "multipart.bin"; + let first_part = vec![b'm'; 5 * 1024 * 1024]; + let second_part = b"reverse multipart tail".to_vec(); + let mut multipart_body = first_part.clone(); + multipart_body.extend_from_slice(&second_part); + + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create reverse decommission bucket"); + let mut source = PutObjReader::from_vec(object_body.clone()); + store.pools[1] + .put_object(&bucket, object, &mut source, &ObjectOptions::default()) + .await + .expect("write ordinary source object to pool 1"); + + let upload = store.pools[1] + .new_multipart_upload(&bucket, multipart_object, &ObjectOptions::default()) + .await + .expect("create source multipart upload in pool 1"); + let mut completed_parts = Vec::with_capacity(2); + for (part_number, bytes) in [(1, first_part.as_slice()), (2, second_part.as_slice())] { + let mut reader = PutObjReader::from_vec(bytes.to_vec()); + let part = store.pools[1] + .put_object_part( + &bucket, + multipart_object, + &upload.upload_id, + part_number, + &mut reader, + &ObjectOptions::default(), + ) + .await + .expect("write source multipart part"); + completed_parts.push(crate::storage_api_contracts::multipart::CompletePart { + part_num: part.part_num, + etag: part.etag, + ..Default::default() + }); + } + store.pools[1] + .clone() + .complete_multipart_upload(&bucket, multipart_object, &upload.upload_id, completed_parts, &ObjectOptions::default()) + .await + .expect("complete source multipart object in pool 1"); + + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[1].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + assert!(store.is_suspended(1).await, "pool 1 must be the reverse decommission source"); + + let commit_barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + object, + crate::set_disk::PutObjectCommitPause::AfterNamespace, + ); + let source_set = store.pools[1].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 1, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + commit_barrier.wait_until_paused().await; + + let delete_barrier = crate::store::object::DeleteAfterObjectLockSnapshotBarrier::install(&bucket); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object(&delete_bucket, object, ObjectOptions::default()) + .await + }); + delete_barrier.wait_until_paused().await; + delete_barrier.release_and_wait_until_namespace_pending().await; + assert!( + !delete_barrier.namespace_acquired() && !delete.is_finished(), + "the reverse target commit must keep DELETE behind the fixed read fence" + ); + delete.abort(); + assert!( + delete + .await + .expect_err("the blocked DELETE should be canceled") + .is_cancelled(), + "canceling the blocked DELETE must not mutate either pool" + ); + drop(delete_barrier); + + commit_barrier.release(); + drop(commit_barrier); + tokio::time::timeout(Duration::from_secs(60), worker) + .await + .expect("reverse ordinary decommission must not self-deadlock on the fixed target set") + .expect("reverse ordinary decommission worker should join") + .expect("reverse ordinary decommission should complete"); + + let mut ordinary_reader = store.pools[0] + .get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read the ordinary object from the fixed target set"); + let mut ordinary_target_body = Vec::new(); + ordinary_reader + .stream + .read_to_end(&mut ordinary_target_body) + .await + .expect("drain the ordinary target body"); + assert_eq!(ordinary_target_body, object_body, "ordinary migration must preserve the full body"); + let ordinary_source_err = store.pools[1] + .get_object_info(&bucket, object, &ObjectOptions::default()) + .await + .expect_err("ordinary source generation must be cleaned after migration"); + assert!(matches!(ordinary_source_err, StorageError::ObjectNotFound(_, _))); + + let multipart_source_set = store.pools[1].get_disks_by_key(multipart_object); + let multipart_store = Arc::clone(&store); + let multipart_bucket = bucket.clone(); + let multipart_worker = tokio::spawn(async move { + multipart_store + .decommission_entry_for_test( + 1, + MetaCacheEntry { + name: multipart_object.to_string(), + ..Default::default() + }, + multipart_bucket, + multipart_source_set, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(60), multipart_worker) + .await + .expect("reverse multipart decommission must not self-deadlock on new or complete") + .expect("reverse multipart decommission worker should join") + .expect("reverse multipart decommission should complete"); + + let target_info = store.pools[0] + .get_object_info(&bucket, multipart_object, &ObjectOptions::default()) + .await + .expect("read migrated multipart metadata from the fixed target set"); + assert!(target_info.is_multipart(), "migration must retain multipart identity"); + let mut multipart_reader = store.pools[0] + .get_object_reader(&bucket, multipart_object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("read migrated multipart object from the fixed target set"); + let mut multipart_target_body = Vec::new(); + multipart_reader + .stream + .read_to_end(&mut multipart_target_body) + .await + .expect("drain the multipart target body"); + assert_eq!(multipart_target_body, multipart_body, "multipart migration must preserve the full body"); + let multipart_source_err = store.pools[1] + .get_object_info(&bucket, multipart_object, &ObjectOptions::default()) + .await + .expect_err("multipart source generation must be cleaned after migration"); + assert!(matches!(multipart_source_err, StorageError::ObjectNotFound(_, _))); + + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn batch_delete_real_path_preserves_source_pool_errors_in_any_pool_order() { + let temp_dir = tempfile::tempdir().expect("create batch delete pool-error store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "batch-delete-pool-errors", &[4, 4])).await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + for source_pool_idx in [0, 1] { + { + let mut pool_meta = store.pool_meta.write().await; + for pool in &mut pool_meta.pools { + pool.decommission = None; + } + } + + let bucket = format!("batch-del-pool-error-{source_pool_idx}-{}", uuid::Uuid::new_v4()); + let object_names = vec![ + format!("third-{source_pool_idx}.bin"), + format!("first-{source_pool_idx}.bin"), + format!("second-{source_pool_idx}.bin"), + ]; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create batch delete pool-error bucket"); + for pool in &store.pools { + for object_name in &object_names { + let mut reader = PutObjReader::from_vec(format!("pool {} {object_name}", pool.pool_idx).into_bytes()); + pool.put_object(&bucket, object_name, &mut reader, &ObjectOptions::default()) + .await + .expect("seed each object in both the source and active pools"); + } + } + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[source_pool_idx].decommission = Some(PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }); + } + assert!( + store.is_suspended(source_pool_idx).await, + "the injected error pool must be the decommission source" + ); + + let expected_errors = [ + StorageError::ErasureWriteQuorum, + StorageError::NamespaceLockQuorumUnavailable { + mode: "delete_objects_commit", + bucket: bucket.clone(), + object: object_names[1].clone(), + required: 3, + achieved: 2, + }, + StorageError::ErasureWriteQuorum, + ]; + let injection = crate::store::object::BatchDeletePoolErrorInjection::install( + &bucket, + source_pool_idx, + object_names.iter().cloned().zip(expected_errors.iter().cloned()).collect(), + ); + let requests = object_names + .iter() + .map(|object_name| ObjectToDelete { + object_name: object_name.clone(), + ..Default::default() + }) + .collect(); + + let (deleted, errors) = store.delete_objects(&bucket, requests, ObjectOptions::default()).await; + + assert_eq!( + injection.observed(), + object_names.len(), + "the source pool must first complete every real delete" + ); + assert_eq!( + errors, + expected_errors.iter().cloned().map(Some).collect::>(), + "a successful pool must not clear a source pool failure at any request index" + ); + assert_eq!( + deleted.iter().map(|object| object.object_name.as_str()).collect::>(), + object_names.iter().map(String::as_str).collect::>(), + "DeleteObjects must preserve request index mapping while aggregating pool failures" + ); + assert!( + deleted.iter().all(|object| object.found), + "the injected source results must retain real delete success data" + ); + + for pool in &store.pools { + for object_name in &object_names { + let error = pool + .get_object_info(&bucket, object_name, &ObjectOptions::default()) + .await + .expect_err("both the active and source pool delete calls must execute"); + assert!( + matches!(error, StorageError::ObjectNotFound(_, _)), + "unexpected residual object: {error:?}" + ); + } + } + drop(injection); + } + + shutdown.cancel(); + } + + #[tokio::test] + #[serial_test::serial(storage_class_env)] + async fn decommission_source_cleanup_holds_hashed_set_lock_across_preflight() { + let temp_dir = tempfile::tempdir().expect("create multi-set decommission cleanup store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store_with_layout( + temp_dir.path(), + "multi-set-decommission-source-cleanup", + &[(2, 4)], + CancellationToken::new(), + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("decom-source-cleanup-lock-{}", uuid::Uuid::new_v4()); + let object = (0..128) + .map(|index| format!("object-{index}.bin")) + .find(|candidate| store.pools[0].get_disks_by_key(candidate).set_index == 1) + .expect("the deterministic object search should select source set 1"); + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create multi-set decommission cleanup bucket"); + let mut source = PutObjReader::from_vec(b"source generation".to_vec()); + store.pools[0] + .put_object(&bucket, &object, &mut source, &ObjectOptions::default()) + .await + .expect("write the source generation to set 1"); + let source_set = store.pools[0].get_disks_by_key(&object); + assert_eq!(source_set.set_index, 1, "the source must not share the fixed set-0 namespace"); + let expected_source_versions = source_set + .load_file_info_versions_exact(&bucket, &object) + .await + .expect("source versions should be readable") + .expect("the source generation should exist"); + + let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, &object); + let cleanup_store = Arc::clone(&store); + let cleanup_bucket = bucket.clone(); + let cleanup_object = object.clone(); + let cleanup = tokio::spawn(async move { + let mutation_fence = cleanup_store + .acquire_decommission_source_cleanup_fence(&cleanup_bucket, &cleanup_object, source_set.as_ref()) + .await?; + crate::data_movement::cleanup_source_entry_if_unchanged( + source_set, + &cleanup_bucket, + &cleanup_object, + &expected_source_versions, + &[], + crate::data_movement::SourceCleanupBucketFence { + object_mutation_fence: Some(&mutation_fence), + ..Default::default() + }, + "test_multi_set_decommission_source_cleanup", + ) + .await + }); + cleanup_barrier.wait_until_paused().await; + + let put_barrier = crate::set_disk::PutObjectCommitBarrier::install( + &bucket, + &object, + crate::set_disk::PutObjectCommitPause::BeforeNamespace, + ); + let mutation_pool = Arc::clone(&store.pools[0]); + let mutation_bucket = bucket.clone(); + let mutation_object = object.clone(); + let replacement = b"replacement generation".to_vec(); + let expected_replacement = replacement.clone(); + let mutation = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(replacement); + mutation_pool + .put_object(&mutation_bucket, &mutation_object, &mut reader, &ObjectOptions::default()) + .await + }); + put_barrier.wait_until_paused().await; + put_barrier.release_and_wait_until_namespace_pending().await; + assert!(!mutation.is_finished(), "a source mutation must wait behind cleanup's set-1 write lock"); + + cleanup_barrier.release(); + cleanup + .await + .expect("source cleanup task should join") + .expect("source cleanup should remove only the preflight generation"); + mutation + .await + .expect("source mutation task should join") + .expect("source mutation should commit after cleanup releases the set lock"); + + let mut reader = store.pools[0] + .get_object_reader(&bucket, &object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("the replacement generation must remain readable"); + let mut actual = Vec::new(); + reader + .stream + .read_to_end(&mut actual) + .await + .expect("read the replacement generation"); + assert_eq!(actual, expected_replacement, "cleanup must not delete the replacement generation"); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn versioned_delete_marker_survives_decommission_source_cleanup() { + let temp_dir = tempfile::tempdir().expect("create versioned decommission delete-fence store dir"); + let (_ctx, store, shutdown) = + without_storage_class_env(build_isolated_test_store(temp_dir.path(), "versioned-decommission-delete-fence", &[4, 4])) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("versioned-decom-delete-{}", uuid::Uuid::new_v4()); + let object = "object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create versioned decommission delete-fence bucket"); + let (source_version, expected_source_versions) = migrate_versioned_decommission_test_object( + &store, + &bucket, + object, + b"source generation", + "test_versioned_decommission_delete_fence", + ) + .await; + + let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object( + &delete_bucket, + object, + ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + }); + delete_barrier.wait_until_paused().await; + let cleanup_set = store.pools[0].get_disks_by_key(object); + crate::data_movement::ensure_source_cleanup_versions_unchanged( + Arc::clone(&cleanup_set), + &bucket, + object, + &expected_source_versions, + &[], + "test_versioned_decommission_delete_fence", + ) + .await + .expect("the committed delete marker must not be published to the suspended source pool"); + + let cleanup_delete_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object); + let cleanup_store = Arc::clone(&store); + let cleanup_bucket = bucket.clone(); + let cleanup = tokio::spawn(async move { + let mutation_fence = cleanup_store + .acquire_decommission_source_cleanup_fence(&cleanup_bucket, object, cleanup_set.as_ref()) + .await?; + crate::data_movement::cleanup_source_entry_if_unchanged( + cleanup_set, + &cleanup_bucket, + object, + &expected_source_versions, + &[], + crate::data_movement::SourceCleanupBucketFence { + object_mutation_fence: Some(&mutation_fence), + ..Default::default() + }, + "test_versioned_decommission_delete_fence", + ) + .await + }); + cleanup_delete_barrier.wait_until_fence_pending().await; + assert!( + !cleanup_delete_barrier.is_paused(), + "source cleanup must wait for the versioned DELETE mutation fence" + ); + + delete_barrier.release(); + let marker = delete + .await + .expect("versioned DELETE task should join") + .expect("versioned DELETE should publish a delete marker after migration"); + assert!(marker.delete_marker, "versioned DELETE must publish a delete marker"); + assert!( + marker.version_id.is_some_and(|version_id| !version_id.is_nil()), + "the delete marker must have a non-nil version ID" + ); + + cleanup_delete_barrier.wait_until_paused().await; + cleanup_delete_barrier.release(); + cleanup + .await + .expect("source cleanup task should join") + .expect("source cleanup should preserve the active-pool delete marker"); + + let err = store + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + ..Default::default() + }, + ) + .await + .expect_err("the post-migration delete marker must hide the migrated version"); + assert!( + matches!(err, StorageError::ObjectNotFound(_, _)), + "unexpected latest-version result: {err:?}" + ); + store + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version.to_string()), + ..Default::default() + }, + ) + .await + .expect("the migrated source version must remain addressable below the delete marker"); + store.pools[0] + .get_object_info( + &bucket, + object, + &ObjectOptions { + versioned: true, + version_id: Some(source_version.to_string()), + ..Default::default() + }, + ) + .await + .expect_err("source cleanup must remove the decommissioned source versions"); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn versioned_batch_delete_marker_skips_decommission_source() { + let temp_dir = tempfile::tempdir().expect("create versioned batch decommission store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "versioned-batch-decommission-delete-fence", + &[4, 4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("vbatch-decom-delete-{}", uuid::Uuid::new_v4()); + let object = "batch-object.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create versioned batch decommission bucket"); + let (_source_version, expected_source_versions) = migrate_versioned_decommission_test_object( + &store, + &bucket, + object, + b"batch source generation", + "test_versioned_batch_decommission_delete_fence", + ) + .await; + + let delete_config_snapshot = + Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test( + s3s::dto::VersioningConfiguration { + status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)), + ..Default::default() + }, + None, + )); + let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_objects( + &delete_bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + ObjectOptions { + delete_replication_config_snapshot: Some(delete_config_snapshot), + ..Default::default() + }, + ) + .await + }); + delete_barrier.wait_until_paused().await; + + let source_set = store.pools[0].get_disks_by_key(object); + crate::data_movement::ensure_source_cleanup_versions_unchanged( + source_set, + &bucket, + object, + &expected_source_versions, + &[], + "test_versioned_batch_decommission_delete_fence", + ) + .await + .expect("batch DELETE must not publish a marker to the suspended source"); + + delete_barrier.release(); + let (deleted, errors) = delete.await.expect("versioned batch DELETE task should join"); + assert!(errors.iter().all(Option::is_none), "versioned batch DELETE should succeed: {errors:?}"); + assert_eq!(deleted.len(), 1); + assert!(deleted[0].delete_marker, "versioned batch DELETE must return a marker"); + assert!( + deleted[0] + .delete_marker_version_id + .is_some_and(|version_id| !version_id.is_nil()), + "versioned batch DELETE marker must have a non-nil version ID" + ); + + let mut active_marker_count = 0; + for pool in store.pools.iter().skip(1) { + let Some(versions) = pool + .get_disks_by_key(object) + .load_file_info_versions_exact(&bucket, object) + .await + .expect("active-pool versions should be readable") + else { + continue; + }; + active_marker_count += versions.versions.iter().filter(|version| version.deleted).count(); + } + assert_eq!(active_marker_count, 1, "batch DELETE must publish exactly one active-pool marker"); + + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn suspended_delete_marker_then_decommission_worker_converges_null_source() { + let temp_dir = tempfile::tempdir().expect("create suspended decommission DELETE store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "suspended-decommission-delete-convergence", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("suspended-decom-delete-{}", uuid::Uuid::new_v4()); + let object = "single.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create suspended decommission DELETE bucket"); + write_suspended_decommission_source(&store, &bucket, object).await; + mark_test_pool_decommissioning(&store, 0).await; + + let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_object( + &delete_bucket, + object, + ObjectOptions { + version_suspended: true, + ..Default::default() + }, + ) + .await + }); + delete_barrier.wait_until_paused().await; + assert_suspended_null_source_present(&store, &bucket, object).await; + + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + delete_barrier.release(); + let marker = delete + .await + .expect("suspended DELETE task should join") + .expect("suspended DELETE should commit its active-pool marker"); + drop(delete_barrier); + assert!(marker.delete_marker, "suspended DELETE must create a marker"); + assert!( + marker.version_id.is_none_or(|version_id| version_id.is_nil()), + "suspended DELETE marker must keep the null version identity" + ); + worker + .await + .expect("suspended decommission worker should join") + .expect("worker must treat the newer active null marker as a completed migration"); + + assert_suspended_decommission_converged(&store, &bucket, object).await; + shutdown.cancel(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[serial_test::serial(storage_class_env)] + async fn suspended_batch_delete_marker_then_decommission_worker_converges_null_source() { + let temp_dir = tempfile::tempdir().expect("create suspended batch decommission DELETE store dir"); + let (_ctx, store, shutdown) = without_storage_class_env(build_isolated_test_store( + temp_dir.path(), + "suspended-batch-decommission-delete-convergence", + &[4, 4], + )) + .await; + crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await; + + let bucket = format!("susp-batch-decom-delete-{}", uuid::Uuid::new_v4()); + let object = "batch.bin"; + store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("create suspended batch decommission DELETE bucket"); + write_suspended_decommission_source(&store, &bucket, object).await; + mark_test_pool_decommissioning(&store, 0).await; + + let delete_config_snapshot = + Arc::new(crate::bucket::replication::DeleteReplicationConfigSnapshot::from_configs_for_test( + s3s::dto::VersioningConfiguration { + status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::SUSPENDED)), + ..Default::default() + }, + None, + )); + let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object); + let delete_store = Arc::clone(&store); + let delete_bucket = bucket.clone(); + let delete = tokio::spawn(async move { + delete_store + .delete_objects( + &delete_bucket, + vec![ObjectToDelete { + object_name: object.to_string(), + ..Default::default() + }], + ObjectOptions { + delete_replication_config_snapshot: Some(delete_config_snapshot), + ..Default::default() + }, + ) + .await + }); + delete_barrier.wait_until_paused().await; + assert_suspended_null_source_present(&store, &bucket, object).await; + + let source_set = store.pools[0].get_disks_by_key(object); + let worker_store = Arc::clone(&store); + let worker_bucket = bucket.clone(); + let worker = tokio::spawn(async move { + worker_store + .decommission_entry_for_test( + 0, + MetaCacheEntry { + name: object.to_string(), + ..Default::default() + }, + worker_bucket, + source_set, + ) + .await + }); + + delete_barrier.release(); + let (deleted, errors) = delete.await.expect("suspended batch DELETE task should join"); + drop(delete_barrier); + assert!(errors.iter().all(Option::is_none), "suspended batch DELETE should succeed: {errors:?}"); + assert!( + matches!(deleted.as_slice(), [marker] if marker.delete_marker && marker.delete_marker_version_id.is_none_or(|version_id| version_id.is_nil())), + "suspended batch DELETE must create one null marker: {deleted:?}" + ); + worker + .await + .expect("suspended batch decommission worker should join") + .expect("worker must treat the newer batch null marker as a completed migration"); + + assert_suspended_decommission_converged(&store, &bucket, object).await; + shutdown.cancel(); + } + #[cfg(feature = "test-util")] #[tokio::test] #[serial_test::serial(storage_class_env)] diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index 757be0c53..a40edd929 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -151,7 +151,7 @@ pub(crate) mod init_format; pub(crate) mod list_objects; mod multipart; mod object; -pub(crate) use object::ObjectLockDiagGuard; +pub(crate) use object::{ObjectLockDiagGuard, SourceCleanupMutationFence}; pub use object::{ PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError, SnapshotConsistencyError, diff --git a/crates/ecstore/src/store/multipart.rs b/crates/ecstore/src/store/multipart.rs index 2c0b18b2a..8d72287d4 100644 --- a/crates/ecstore/src/store/multipart.rs +++ b/crates/ecstore/src/store/multipart.rs @@ -400,7 +400,7 @@ impl ECStore { object: &str, opts: &ObjectOptions, ) -> Result { - self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts) + self.handle_new_multipart_upload_with_pool_idx(bucket, object, opts, None) .await .map(|(res, _, _)| res) } @@ -410,20 +410,22 @@ impl ECStore { bucket: &str, object: &str, opts: &ObjectOptions, + mutation_fence: Option<&ObjectLockDiagGuard>, ) -> Result<(MultipartUploadResult, usize, Option)> { check_new_multipart_args(bucket, object)?; - let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; - let opts = &opts; + let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?; if self.single_pool() { + self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence) + .await; return self.pools[0] - .new_multipart_upload(bucket, object, opts) + .new_multipart_upload(bucket, object, &opts) .await .map(|res| (res, 0, opts.expected_bucket_incarnation_id)); } if opts.data_movement && opts.version_id.is_some() { - let idx = self.select_data_movement_pool_idx(bucket, object, -1, opts, false).await?; + let idx = self.select_data_movement_pool_idx(bucket, object, -1, &opts, false).await?; if idx == opts.src_pool_idx { return Err(StorageError::DataMovementOverwriteErr( bucket.to_owned(), @@ -431,7 +433,9 @@ impl ECStore { opts.version_id.clone().unwrap_or_default(), )); } - let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; + self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence) + .await; + let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?; return Ok((res, idx, opts.expected_bucket_incarnation_id)); } @@ -454,7 +458,9 @@ impl ECStore { .await?; if !res.uploads.is_empty() { - let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; + self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence) + .await; + let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?; return Ok((res, idx, opts.expected_bucket_incarnation_id)); } } @@ -467,7 +473,9 @@ impl ECStore { )); } - let res = self.pools[idx].new_multipart_upload(bucket, object, opts).await?; + self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence) + .await; + let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?; Ok((res, idx, opts.expected_bucket_incarnation_id)) } @@ -704,13 +712,14 @@ impl ECStore { pub(crate) async fn complete_multipart_upload_for_data_movement( self: Arc, - target_pool_idx: usize, + target: (usize, Option<&ObjectLockDiagGuard>), bucket: &str, object: &str, upload_id: &str, uploaded_parts: Vec, opts: &ObjectOptions, ) -> Result { + let (target_pool_idx, mutation_fence) = target; check_complete_multipart_args(bucket, object, upload_id)?; if !opts.data_movement { return Err(Error::other("targeted multipart completion requires data_movement options")); @@ -739,6 +748,8 @@ impl ECStore { snapshot.add_lock_fences(&mut opts); opts.object_lock_config_snapshot = Some(snapshot); } + self.apply_decommission_target_mutation_fence(target_pool_idx, object, &mut opts, mutation_fence) + .await; #[cfg(test)] pause_data_movement_multipart_before_selected_completion(bucket).await; let pool = self diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index e5f2f0465..79a8232f7 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -32,12 +32,13 @@ use crate::bucket::metadata_sys::{ use crate::bucket::object_lock::objectlock_sys::{ check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state, }; -use crate::bucket::replication::ReplicationObjectBridge; +use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge}; +use crate::bucket::versioning::VersioningApi; use crate::disk::OldCurrentSize; use crate::object_api::{NamespaceLockFence, ObjectLockConfigSnapshot}; use crate::set_disk::{ - get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, - is_lock_optimization_enabled, is_object_lock_diag_enabled, + SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold, + is_lock_optimization_enabled, is_object_lock_diag_enabled, same_distributed_lock_domain, }; use crate::storage_api_contracts::{ namespace::NamespaceLocking as _, @@ -352,6 +353,8 @@ impl fmt::Display for ObjectLockDiagMode { pub(crate) struct ObjectLockDiagGuard { guard: rustfs_lock::NamespaceLockGuard, + #[cfg(test)] + test_namespace_lock_fence: Option, enabled: bool, op: &'static str, bucket: Option, @@ -373,6 +376,8 @@ impl ObjectLockDiagGuard { ) -> Self { Self { guard, + #[cfg(test)] + test_namespace_lock_fence: None, enabled, op, bucket, @@ -393,6 +398,115 @@ impl ObjectLockDiagGuard { pub(crate) fn is_lock_lost(&self) -> bool { self.guard.is_lock_lost() } + + pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { + opts.ensure_namespace_lock_fence(); + if let Some(signal) = self.lock_lost_signal() { + opts.add_namespace_lock_lost_signal(signal); + } + #[cfg(test)] + if let Some(fence) = self.test_namespace_lock_fence.as_ref() { + opts.add_namespace_lock_fence_for_test(fence); + } + } +} + +#[cfg(test)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum DecommissionMutationFenceTestPhase { + Migration, + SourceCleanup, +} + +#[cfg(test)] +struct DecommissionMutationFenceLossState { + bucket: String, + object: String, + phase: DecommissionMutationFenceTestPhase, + fence: NamespaceLockFence, + loss_handle: Arc, +} + +#[cfg(test)] +pub(crate) struct DecommissionMutationFenceLossHook { + state: Arc, +} + +#[cfg(test)] +static DECOMMISSION_MUTATION_FENCE_LOSS_HOOK: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl DecommissionMutationFenceLossHook { + pub(crate) fn install(bucket: &str, object: &str, phase: DecommissionMutationFenceTestPhase) -> Self { + let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test(); + let state = Arc::new(DecommissionMutationFenceLossState { + bucket: bucket.to_string(), + object: object.to_string(), + phase, + fence, + loss_handle, + }); + let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission mutation fence loss hooks should not poison"); + assert!(slot.is_none(), "decommission mutation fence loss hook must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) fn mark_lost(&self) { + self.state.loss_handle.store(true, Ordering::Release); + } +} + +#[cfg(test)] +impl Drop for DecommissionMutationFenceLossHook { + fn drop(&mut self) { + let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission mutation fence loss hooks should not poison"); + if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +fn decommission_mutation_fence_for_test( + bucket: &str, + object: &str, + phase: DecommissionMutationFenceTestPhase, +) -> Option { + DECOMMISSION_MUTATION_FENCE_LOSS_HOOK + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("decommission mutation fence loss hooks should not poison") + .as_ref() + .filter(|hook| hook.bucket == bucket && hook.object == object && hook.phase == phase) + .map(|hook| hook.fence.clone()) +} + +pub(crate) struct SourceCleanupMutationFence { + guard: ObjectLockDiagGuard, + source_lock_covered: bool, +} + +impl SourceCleanupMutationFence { + pub(crate) fn source_lock_covered(&self) -> bool { + self.source_lock_covered + } + + pub(crate) fn is_lock_lost(&self) -> bool { + self.guard.is_lock_lost() + } + + pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { + self.guard.add_namespace_lock_fence(opts); + } } /// Opaque write-lock guard for the RestoreObject accept path; see @@ -410,10 +524,7 @@ impl RestoreAcceptGuard { } pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) { - opts.ensure_namespace_lock_fence(); - if let Some(signal) = self.0.lock_lost_signal() { - opts.add_namespace_lock_lost_signal(signal); - } + self.0.add_namespace_lock_fence(opts); } } @@ -690,16 +801,6 @@ impl SelectObjectSnapshotLockLossWake { } } -// LockRegistry clones its canonical client Arc for each endpoint host, so an -// exact Arc set identifies one distributed namespace-lock quorum domain. -fn same_distributed_lock_domain(left: &[Arc], right: &[Arc]) -> bool { - left.iter() - .all(|left_client| right.iter().any(|right_client| Arc::ptr_eq(left_client, right_client))) - && right - .iter() - .all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client))) -} - impl AsyncRead for SelectObjectSnapshotReader { fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { if self.lock_loss_wake.poll_lost(cx) || self.lease.is_lost() { @@ -805,7 +906,7 @@ fn resolve_latest_object_access( } fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool { - opts.versioned && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement + (opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement } #[cfg(test)] @@ -813,6 +914,8 @@ struct DeleteAfterObjectLockSnapshotBarrierState { bucket: String, arrived: tokio::sync::Notify, release: tokio::sync::Notify, + namespace_pending: tokio::sync::Notify, + namespace_acquired: AtomicBool, } #[cfg(test)] @@ -832,6 +935,8 @@ impl DeleteAfterObjectLockSnapshotBarrier { bucket: bucket.to_string(), arrived: tokio::sync::Notify::new(), release: tokio::sync::Notify::new(), + namespace_pending: tokio::sync::Notify::new(), + namespace_acquired: AtomicBool::new(false), }); let mut slot = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER .get_or_init(|| std::sync::Mutex::new(None)) @@ -849,6 +954,18 @@ impl DeleteAfterObjectLockSnapshotBarrier { pub(crate) fn release(&self) { self.state.release.notify_one(); } + + pub(crate) async fn release_and_wait_until_namespace_pending(&self) { + let namespace_pending = self.state.namespace_pending.notified(); + self.release(); + tokio::time::timeout(Duration::from_secs(5), namespace_pending) + .await + .expect("delete should proceed to its namespace lock after leaving the snapshot barrier"); + } + + pub(crate) fn namespace_acquired(&self) -> bool { + self.state.namespace_acquired.load(Ordering::Acquire) + } } #[cfg(test)] @@ -873,6 +990,97 @@ async fn pause_delete_after_object_lock_snapshot(bucket: &str) { .as_ref() .filter(|state| state.bucket == bucket) .cloned(); + if let Some(state) = state { + state.arrived.notify_one(); + state.release.notified().await; + state.namespace_pending.notify_one(); + } +} + +#[cfg(test)] +fn notify_delete_namespace_acquired(bucket: &str) { + let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("delete snapshot barrier mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket) + .cloned(); + if let Some(state) = state { + state.namespace_acquired.store(true, Ordering::Release); + } +} + +#[cfg(test)] +struct VersionedDeleteMarkerCommitBarrierState { + bucket: String, + object: String, + arrived: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +#[cfg(test)] +pub(crate) struct VersionedDeleteMarkerCommitBarrier { + state: Arc, +} + +#[cfg(test)] +static VERSIONED_DELETE_MARKER_COMMIT_BARRIER: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl VersionedDeleteMarkerCommitBarrier { + pub(crate) fn install(bucket: &str, object: &str) -> Self { + let state = Arc::new(VersionedDeleteMarkerCommitBarrierState { + bucket: bucket.to_string(), + object: object.to_string(), + arrived: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("versioned delete-marker commit barrier mutex should not poison"); + assert!(slot.is_none(), "versioned delete-marker commit barrier must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) async fn wait_until_paused(&self) { + tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified()) + .await + .expect("versioned DELETE should reach the post-marker-commit barrier"); + } + + pub(crate) fn release(&self) { + self.state.release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for VersionedDeleteMarkerCommitBarrier { + fn drop(&mut self) { + self.state.release.notify_one(); + let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("versioned delete-marker commit barrier mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +async fn pause_versioned_delete_marker_after_commit(bucket: &str, object: &str) { + let state = VERSIONED_DELETE_MARKER_COMMIT_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("versioned delete-marker commit barrier mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket && state.object == object) + .cloned(); if let Some(state) = state { state.arrived.notify_one(); state.release.notified().await; @@ -913,6 +1121,160 @@ fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions lookup_opts } +fn delete_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions { + let mut lookup_opts = writer_pool_lookup_opts(opts, no_lock); + lookup_opts.skip_decommissioned = opts.data_movement; + lookup_opts +} + +fn should_delete_from_all_pools(opts: &ObjectOptions, pool_count: usize) -> bool { + pool_count > 0 && (!opts.versioned && !opts.version_suspended || opts.version_id.is_some()) +} + +fn batch_delete_creates_latest_marker(object: &ObjectToDelete, delete_config_snapshot: &DeleteReplicationConfigSnapshot) -> bool { + if object.version_id.is_some() { + return false; + } + + let object_name = decode_dir_object(&object.object_name); + let (versioned, version_suspended) = delete_config_snapshot.versioning_config().delete_state(&object_name); + versioned || version_suspended +} + +fn batch_delete_targets_pool(creates_latest_marker: bool, marker_target_pool_idx: Option, pool_idx: usize) -> bool { + !creates_latest_marker || marker_target_pool_idx == Some(pool_idx) +} + +#[cfg(test)] +struct BatchDeletePoolErrorInjectionState { + bucket: String, + pool_idx: usize, + errors: std::collections::HashMap, + observed: std::sync::atomic::AtomicUsize, +} + +#[cfg(test)] +pub(crate) struct BatchDeletePoolErrorInjection { + state: Arc, +} + +#[cfg(test)] +static BATCH_DELETE_POOL_ERROR_INJECTION: std::sync::OnceLock>>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +impl BatchDeletePoolErrorInjection { + pub(crate) fn install(bucket: &str, pool_idx: usize, errors: Vec<(String, Error)>) -> Self { + let state = Arc::new(BatchDeletePoolErrorInjectionState { + bucket: bucket.to_string(), + pool_idx, + errors: errors.into_iter().collect(), + observed: std::sync::atomic::AtomicUsize::new(0), + }); + let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("batch delete pool error injection mutex should not poison"); + assert!(slot.is_none(), "batch delete pool error injection must be unique"); + *slot = Some(Arc::clone(&state)); + Self { state } + } + + pub(crate) fn observed(&self) -> usize { + self.state.observed.load(Ordering::Acquire) + } +} + +#[cfg(test)] +impl Drop for BatchDeletePoolErrorInjection { + fn drop(&mut self) { + let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("batch delete pool error injection mutex should not poison"); + if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *slot = None; + } + } +} + +#[cfg(test)] +fn inject_batch_delete_pool_errors( + bucket: &str, + pool_idx: usize, + object_names: &[String], + result: &mut (Vec, Vec>), +) { + let state = BATCH_DELETE_POOL_ERROR_INJECTION + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("batch delete pool error injection mutex should not poison") + .as_ref() + .filter(|state| state.bucket == bucket && state.pool_idx == pool_idx) + .cloned(); + let Some(state) = state else { + return; + }; + + for (idx, object_name) in object_names.iter().enumerate() { + let Some(error) = state.errors.get(object_name) else { + continue; + }; + if result.1[idx].is_none() && result.0[idx].found { + result.1[idx] = Some(error.clone()); + state.observed.fetch_add(1, Ordering::AcqRel); + } + } +} + +fn resolve_batch_delete_pool_results<'a>( + initial_error: Option, + pool_results: impl IntoIterator)>, +) -> (Option, Option, bool) { + let mut failure = initial_error.map(|err| (None, err)); + let mut deleted = None; + let mut fallback: Option<(DeletedObject, Option)> = None; + let mut attempted = false; + + for (pool_delete, pool_error) in pool_results { + attempted = true; + match pool_error { + Some(err) if is_err_object_not_found(err) || is_err_version_not_found(err) => { + if fallback.as_ref().is_none_or(|(_, error)| error.is_none()) { + fallback = Some(((*pool_delete).clone(), Some(err.clone()))); + } + } + Some(err) => { + if failure.is_none() { + failure = Some((Some((*pool_delete).clone()), err.clone())); + } + } + None if pool_delete.found => { + if deleted.is_none() { + deleted = Some((*pool_delete).clone()); + } + } + None => { + if fallback.is_none() { + fallback = Some(((*pool_delete).clone(), None)); + } + } + } + } + + if let Some((failed_delete, err)) = failure { + return (failed_delete, Some(err), attempted); + } + if let Some(deleted) = deleted { + return (Some(deleted), None, attempted); + } + if let Some((deleted, err)) = fallback { + return (Some(deleted), err, attempted); + } + + (None, None, attempted) +} + fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions { let mut lookup_opts = opts.clone(); lookup_opts.skip_decommissioned = true; @@ -1541,6 +1903,89 @@ impl ECStore { ))) } + pub(crate) async fn acquire_decommission_object_mutation_fence( + &self, + bucket: &str, + object: &str, + ) -> Result { + if self.ctx.lock_manager().is_disabled() { + return Err(Error::other("decommission object migration requires namespace locking")); + } + + #[cfg(test)] + let test_namespace_lock_fence = + decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::Migration); + let object = encode_dir_object(object); + let mut opts = ObjectOptions::default(); + let guard = self + .acquire_object_read_lock_if_needed("decommission_object", bucket, &object, &mut opts) + .await? + .ok_or_else(|| Error::other("decommission object migration failed to acquire its namespace fence"))?; + #[cfg(test)] + let guard = { + let mut guard = guard; + guard.test_namespace_lock_fence = test_namespace_lock_fence; + guard + }; + Ok(guard) + } + + pub(super) async fn apply_decommission_target_mutation_fence( + &self, + target_pool_idx: usize, + object: &str, + opts: &mut ObjectOptions, + mutation_fence: Option<&ObjectLockDiagGuard>, + ) { + let Some(mutation_fence) = mutation_fence else { + return; + }; + + mutation_fence.add_namespace_lock_fence(opts); + let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()); + let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object)); + opts.no_lock = match (fixed_set, target_set) { + (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await, + _ => false, + }; + } + + pub(crate) async fn acquire_decommission_source_cleanup_fence( + &self, + bucket: &str, + object: &str, + source_set: &SetDisks, + ) -> Result { + if self.ctx.lock_manager().is_disabled() { + return Err(Error::other("decommission source cleanup requires namespace locking")); + } + + #[cfg(test)] + crate::data_movement::notify_source_cleanup_mutation_fence_pending(bucket, object); + #[cfg(test)] + let test_namespace_lock_fence = + decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::SourceCleanup); + let object = encode_dir_object(object); + let fixed_set = Arc::clone(&self.pools[0].disk_set[0]); + let source_lock_covered = fixed_set.shares_namespace_lock_domain(source_set).await; + // Lock order: fixed store mutation domain first; source cleanup takes its + // hashed source-domain lock second only when this guard does not cover it. + let guard = self + .acquire_object_write_lock("decommission_source_cleanup", bucket, &object) + .await?; + #[cfg(test)] + let guard = { + let mut guard = guard; + guard.test_namespace_lock_fence = test_namespace_lock_fence; + guard + }; + + Ok(SourceCleanupMutationFence { + guard, + source_lock_covered, + }) + } + pub(crate) async fn acquire_all_object_read_locks( &self, op: &'static str, @@ -1994,14 +2439,17 @@ impl ECStore { object: &str, data: &mut PutObjReader, opts: &ObjectOptions, + mutation_fence: Option<&ObjectLockDiagGuard>, ) -> Result<(usize, Result)> { if !opts.data_movement { return Err(Error::other("data movement PUT requires data_movement options")); } - let (object, opts) = self.prepare_put_object(bucket, object, opts).await?; + let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?; let idx = self .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts) .await?; + self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence) + .await; let result = self.pools[idx] .put_object_with_old_current_size(bucket, &object, data, &opts) .await @@ -2470,6 +2918,10 @@ impl ECStore { } else { None }; + #[cfg(test)] + if _object_lock_guard.is_some() { + notify_delete_namespace_acquired(bucket); + } if let Some(trigger) = opts.lifecycle_delete_all.as_ref() { let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?; let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?; @@ -2503,7 +2955,7 @@ impl ECStore { return Ok(ObjectInfo::default()); } - let gopts = writer_pool_lookup_opts(&opts, true); + let gopts = delete_pool_lookup_opts(&opts, true); if opts.data_movement { let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await; @@ -2608,6 +3060,8 @@ impl ECStore { Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => { let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?; let mut obj = self.pools[target_pool_idx].delete_object(bucket, object, opts).await?; + #[cfg(test)] + pause_versioned_delete_marker_after_commit(bucket, object).await; obj.name = decode_dir_object(object); return Ok(obj); } @@ -2646,7 +3100,7 @@ impl ECStore { None }; - if !errs.is_empty() && !opts.versioned && !opts.version_suspended { + if should_delete_from_all_pools(&opts, errs.len()) { let mut obj = match self.delete_object_from_all_pools(bucket, object, &opts, errs).await { Ok(obj) => obj, Err(err) => { @@ -2670,6 +3124,8 @@ impl ECStore { match pool.delete_object(bucket, object, opts.clone()).await { Ok(res) => { + #[cfg(test)] + pause_versioned_delete_marker_after_commit(bucket, object).await; if let (Some(api), Some(je)) = (tier_journal_api.as_ref(), journal_entry.as_ref()) { commit_prepared_tier_delete_journal_entry(api, je).await; } @@ -2817,32 +3273,104 @@ impl ECStore { Ok(guards) => guards, Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err), }; + #[cfg(test)] + if !_object_lock_guards.is_empty() { + notify_delete_namespace_acquired(bucket); + } + + let delete_config_snapshot = opts + .delete_replication_config_snapshot + .as_deref() + .expect("batch delete replication config snapshot should be loaded"); + let latest_marker_objects = objects + .iter() + .map(|object| batch_delete_creates_latest_marker(object, delete_config_snapshot)) + .collect::>(); + let marker_target_results = join_all(objects.iter().zip(&latest_marker_objects).map( + |(object, creates_marker)| async move { + if *creates_marker { + Some(self.get_pool_idx_no_lock(bucket, &object.object_name, 0).await) + } else { + None + } + }, + )) + .await; + let mut marker_target_pool_indices = Vec::with_capacity(objects.len()); + for (idx, target_result) in marker_target_results.into_iter().enumerate() { + match target_result { + Some(Ok(pool_idx)) => marker_target_pool_indices.push(Some(pool_idx)), + Some(Err(err)) => { + del_errs[idx] = Some(err); + marker_target_pool_indices.push(None); + } + None => marker_target_pool_indices.push(None), + } + } let mut futures = Vec::with_capacity(self.pools.len()); - for pool in self.pools.iter() { if self.is_pool_rebalancing(pool.pool_idx).await { continue; } - futures.push(pool.delete_objects_with_accounting(bucket, objects.clone(), opts.clone())); + + let (object_indices, pool_objects): (Vec<_>, Vec<_>) = objects + .iter() + .enumerate() + .filter(|(idx, _)| { + batch_delete_targets_pool(latest_marker_objects[*idx], marker_target_pool_indices[*idx], pool.pool_idx) + }) + .map(|(idx, object)| (idx, object.clone())) + .unzip(); + if pool_objects.is_empty() { + continue; + } + + let pool_opts = opts.clone(); + futures.push(async move { + #[cfg(test)] + let pool_object_names = pool_objects + .iter() + .map(|object| object.object_name.clone()) + .collect::>(); + let result = pool.delete_objects(bucket, pool_objects, pool_opts).await; + #[cfg(test)] + let result = { + let mut result = result; + inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, &mut result); + result + }; + (object_indices, result) + }); } let results = join_all(futures).await; for idx in 0..del_objects.len() { - for (dels, errs, pool_accounting) in results.iter() { - if errs[idx].is_none() && dels[idx].found { - del_errs[idx] = None; - del_objects[idx] = dels[idx].clone(); - accounting[idx] = pool_accounting[idx].clone(); - break; - } + let pool_results = results.iter().filter_map(|(object_indices, (dels, errs))| { + let pool_object_idx = object_indices.binary_search(&idx).ok()?; + Some((&dels[pool_object_idx], &errs[pool_object_idx])) + }); + let (deleted, error, attempted) = resolve_batch_delete_pool_results(del_errs[idx].take(), pool_results); + if let Some(deleted) = deleted { + del_objects[idx] = deleted; + } + del_errs[idx] = error; - if del_errs[idx].is_none() { - del_errs[idx] = errs[idx].clone(); - del_objects[idx] = dels[idx].clone(); - accounting[idx] = pool_accounting[idx].clone(); - } + if !attempted && del_errs[idx].is_none() && latest_marker_objects[idx] { + del_objects[idx] = DeletedObject { + object_name: objects[idx].object_name.clone(), + version_id: objects[idx].version_id, + ..Default::default() + }; + del_errs[idx] = Some(StorageError::ObjectNotFound(bucket.to_owned(), objects[idx].object_name.clone())); + } + } + + #[cfg(test)] + for (idx, object) in objects.iter().enumerate() { + if del_errs[idx].is_none() && del_objects[idx].delete_marker { + pause_versioned_delete_marker_after_commit(bucket, &object.object_name).await; } } @@ -3417,6 +3945,80 @@ mod tests { assert!(!same_distributed_lock_domain(&[first, second], &[other])); } + #[tokio::test] + async fn decommission_fence_covers_dist_sets_with_same_clients_despite_different_namespaces() { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_dirs, original_sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await; + let mut second_set = (*original_sets.disk_set[1]).clone(); + second_set.lockers = original_sets.disk_set[0].lockers.clone(); + let mut sets = (*original_sets).clone(); + sets.disk_set[1] = Arc::new(second_set); + let sets = Arc::new(sets); + ctx.update_erasure_type(SetupType::DistErasure).await; + + assert!( + sets.disk_set[0] + .lockers + .iter() + .zip(&sets.disk_set[1].lockers) + .all(|(fixed, hashed)| Arc::ptr_eq(fixed, hashed)), + "the regression requires identical distributed lock clients" + ); + assert_ne!(sets.disk_set[0].set_index, sets.disk_set[1].set_index); + + let pool_config = sets.endpoints.clone(); + let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx); + let object = (0..1_000) + .map(|index| format!("decommission-dist-domain-{index}.bin")) + .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1])) + .expect("a key should hash to the second set namespace"); + let mutation_fence = store + .acquire_decommission_object_mutation_fence("bucket", &object) + .await + .expect("the fixed distributed mutation fence should be acquired"); + let target_lock = sets.disk_set[1] + .new_ns_lock("bucket", &object) + .await + .expect("the hashed-set namespace lock should be created"); + let target_err = target_lock + .get_write_lock(Duration::from_millis(50)) + .await + .expect_err("the fixed read fence must conflict through the shared clients"); + assert!(matches!(target_err, rustfs_lock::LockError::Timeout { .. })); + + let mut put_opts = ObjectOptions::default(); + store + .apply_decommission_target_mutation_fence(0, &object, &mut put_opts, Some(&mutation_fence)) + .await; + assert!(put_opts.no_lock, "migration target PUT must reuse the covering fixed fence"); + + let mut multipart_opts = ObjectOptions::default(); + store + .apply_decommission_target_mutation_fence(0, &object, &mut multipart_opts, Some(&mutation_fence)) + .await; + assert!(multipart_opts.no_lock, "migration target multipart must reuse the covering fixed fence"); + drop(mutation_fence); + + let cleanup_object = (0..1_000) + .map(|index| format!("decommission-dist-cleanup-{index}.bin")) + .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1])) + .expect("a cleanup key should hash to the second set namespace"); + let source_fence = store + .acquire_decommission_source_cleanup_fence("bucket", &cleanup_object, sets.disk_set[1].as_ref()) + .await + .expect("the fixed distributed cleanup fence should be acquired"); + assert!(source_fence.source_lock_covered(), "source cleanup must reuse the covering fixed fence"); + let source_lock = sets.disk_set[1] + .new_ns_lock("bucket", &cleanup_object) + .await + .expect("the source-set namespace lock should be created"); + let source_err = source_lock + .get_read_lock(Duration::from_millis(50)) + .await + .expect_err("the fixed write fence must conflict through the shared clients"); + assert!(matches!(source_err, rustfs_lock::LockError::Timeout { .. })); + } + #[test] fn select_snapshot_version_matching_normalizes_null_and_uuid_forms() { let nil = Uuid::nil(); @@ -4476,6 +5078,159 @@ mod tests { assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1")); } + #[test] + fn ordinary_delete_lookup_includes_decommission_source_and_skips_rebalance_source() { + let lookup_opts = delete_pool_lookup_opts(&ObjectOptions::default(), true); + + assert!(lookup_opts.no_lock); + assert!(!lookup_opts.skip_decommissioned); + assert!(lookup_opts.skip_rebalancing); + + let explicit_version = delete_pool_lookup_opts( + &ObjectOptions { + versioned: true, + version_id: Some(uuid::Uuid::new_v4().to_string()), + ..Default::default() + }, + true, + ); + assert!(!explicit_version.skip_decommissioned); + } + + #[test] + fn delete_fans_out_for_unversioned_and_explicit_version_mutations() { + assert!(should_delete_from_all_pools(&ObjectOptions::default(), 1)); + assert!(should_delete_from_all_pools( + &ObjectOptions { + versioned: true, + version_id: Some(uuid::Uuid::new_v4().to_string()), + ..Default::default() + }, + 2, + )); + assert!(!should_delete_from_all_pools( + &ObjectOptions { + versioned: true, + ..Default::default() + }, + 1, + )); + assert!(!should_delete_from_all_pools(&ObjectOptions::default(), 0)); + } + + #[test] + fn batch_delete_identifies_only_latest_versioned_markers() { + let versioned = DeleteReplicationConfigSnapshot::from_configs_for_test( + s3s::dto::VersioningConfiguration { + status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)), + ..Default::default() + }, + None, + ); + let latest = ObjectToDelete { + object_name: "latest".to_string(), + ..Default::default() + }; + assert!(batch_delete_creates_latest_marker(&latest, &versioned)); + assert!(!batch_delete_targets_pool(true, Some(1), 0)); + assert!(batch_delete_targets_pool(true, Some(1), 1)); + assert!(!batch_delete_targets_pool(true, Some(1), 2)); + + let explicit = ObjectToDelete { + object_name: "explicit".to_string(), + version_id: Some(uuid::Uuid::new_v4()), + ..Default::default() + }; + assert!(!batch_delete_creates_latest_marker(&explicit, &versioned)); + assert!(batch_delete_targets_pool(false, Some(1), 0)); + + let unversioned = DeleteReplicationConfigSnapshot::default(); + assert!(!batch_delete_creates_latest_marker(&latest, &unversioned)); + assert!(batch_delete_targets_pool(false, None, 0)); + } + + #[test] + fn batch_delete_pool_failures_override_success_in_any_pool_order() { + let success = DeletedObject { + object_name: "object".to_string(), + found: true, + ..Default::default() + }; + let source_errors = [ + StorageError::ErasureWriteQuorum, + StorageError::NamespaceLockQuorumUnavailable { + mode: "delete_objects_commit", + bucket: "bucket".to_string(), + object: "object".to_string(), + required: 1, + achieved: 0, + }, + ]; + + for source_error in source_errors { + for source_first in [true, false] { + let failed = (DeletedObject::default(), Some(source_error.clone())); + let succeeded = (success.clone(), None); + let pool_results = if source_first { + vec![failed, succeeded] + } else { + vec![succeeded, failed] + }; + + let (_, error, attempted) = + resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error))); + + assert!(attempted); + assert_eq!(error, Some(source_error.clone())); + } + } + } + + #[test] + fn batch_delete_ignores_missing_pool_only_after_another_pool_succeeds() { + let success = DeletedObject { + object_name: "object".to_string(), + found: true, + ..Default::default() + }; + let missing_errors = [ + StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()), + StorageError::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string()), + ]; + + for missing_error in missing_errors { + let missing = (DeletedObject::default(), Some(missing_error.clone())); + for missing_first in [true, false] { + let succeeded = (success.clone(), None); + let pool_results = if missing_first { + vec![missing.clone(), succeeded] + } else { + vec![succeeded, missing.clone()] + }; + let (deleted, error, attempted) = + resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error))); + + assert!(attempted); + let deleted = deleted.expect("successful pool result should be retained"); + assert!(deleted.found); + assert_eq!(deleted.object_name, success.object_name.as_str()); + assert!(error.is_none()); + } + + let missing_only = [missing]; + let (_, error, attempted) = + resolve_batch_delete_pool_results(None, missing_only.iter().map(|(deleted, error)| (deleted, error))); + assert!(attempted); + assert_eq!(error, Some(missing_error)); + } + + let silent_missing = [(DeletedObject::default(), None)]; + let (_, error, attempted) = + resolve_batch_delete_pool_results(None, silent_missing.iter().map(|(deleted, error)| (deleted, error))); + assert!(attempted); + assert!(error.is_none()); + } + #[test] fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() { let lookup_opts = data_movement_pool_lookup_opts( diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index 7d7c1d91e..dc2fa2ee5 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -859,6 +859,7 @@ fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAl #[cfg(test)] mod tests { use super::*; + use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType}; use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env}; use crate::disk::error::DiskError; use crate::layout::endpoint::Endpoint; @@ -1423,6 +1424,14 @@ mod tests { } } + fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option) -> ObjectInfo { + ObjectInfo { + version_id: Some(version_id), + etag, + ..object_info_with_mod_time(unix_ts, delete_marker) + } + } + #[test] fn resolve_latest_object_info_candidates_returns_latest_delete_marker() { let candidates = vec![ @@ -1446,7 +1455,7 @@ mod tests { } #[test] - fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time() { + fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() { let candidates = vec![ LatestObjectInfoCandidate { info: Some(object_info_with_mod_time(10, false)), @@ -1466,6 +1475,382 @@ mod tests { assert_eq!(idx, 1); } + #[test] + fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 2, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 7, + err: None, + }, + ]; + + let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect("equivalent replicas must resolve deterministically"); + + assert_eq!(idx, 7); + assert_eq!(info.version_id, Some(Uuid::from_u128(1))); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))), + idx: 1, + err: None, + }, + ]; + + let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect_err("divergent version ids must not silently resolve to the higher pool index"); + + assert_eq!(err, Error::ErasureReadQuorum); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))), + idx: 1, + err: None, + }, + ]; + + let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect_err("divergent etags must not silently resolve to the higher pool index"); + + assert_eq!(err, Error::ErasureReadQuorum); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 1, + err: None, + }, + ]; + + let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect_err("a delete marker tied with a live version must not be masked by the pool index"); + + assert_eq!(err, Error::ErasureReadQuorum); + } + + fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) { + let err = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(left), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(right), + idx: 1, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect_err("equal-time identity divergence must fail closed"); + + assert_eq!(err, Error::ErasureReadQuorum); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + + let mut data_dir = base.clone(); + data_dir.data_dir = Some(Uuid::from_u128(2)); + assert_equal_time_identity_conflict(base.clone(), data_dir); + + let mut size = base.clone(); + size.size = 1; + assert_equal_time_identity_conflict(base.clone(), size); + + let mut actual_size = base.clone(); + actual_size.actual_size = 1; + assert_equal_time_identity_conflict(base.clone(), actual_size); + + let mut checksum = base.clone(); + checksum.checksum = Some(bytes::Bytes::from_static(b"checksum")); + assert_equal_time_identity_conflict(base.clone(), checksum); + + let mut parts = base.clone(); + parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo { + etag: "part-etag".to_string(), + number: 1, + size: 1, + ..Default::default() + }]); + assert_equal_time_identity_conflict(base.clone(), parts); + + let mut transition = base; + transition.transitioned_object.tier = "tier-a".to_string(); + assert_equal_time_identity_conflict( + object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())), + transition, + ); + } + + #[test] + fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + let mut rustfs_alias = base.clone(); + rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "x-rustfs-internal-compression".to_string(), + "zstd".to_string(), + )])); + let mut minio_alias = base.clone(); + minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "X-MINIO-INTERNAL-COMPRESSION".to_string(), + "zstd".to_string(), + )])); + + let (_, idx) = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(rustfs_alias), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(minio_alias), + idx: 1, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect("same-value internal aliases should resolve"); + assert_eq!(idx, 1); + + let mut dual_alias = base.clone(); + dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([ + ("x-rustfs-internal-compression".to_string(), "zstd".to_string()), + ("x-minio-internal-compression".to_string(), "zstd".to_string()), + ])); + let mut single_alias = base; + single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "x-rustfs-internal-compression".to_string(), + "zstd".to_string(), + )])); + + let (_, idx) = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(dual_alias), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(single_alias), + idx: 1, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect("dual-key and single-key internal metadata should resolve"); + assert_eq!(idx, 1); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + let mut rustfs_alias = base.clone(); + rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "x-rustfs-internal-compression".to_string(), + "zstd".to_string(), + )])); + let mut minio_alias = base; + minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + "x-minio-internal-compression".to_string(), + "snappy".to_string(), + )])); + + assert_equal_time_identity_conflict(rustfs_alias, minio_alias); + } + + #[test] + fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() { + for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + let mut rustfs_alias = base.clone(); + rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + format!( + "X-RUSTFS-INTERNAL-{}{suffix}", + suffix_prefix.to_uppercase(), + suffix = "arn:aws:s3:::Bucket" + ), + "value".to_string(), + )])); + let mut minio_alias = base.clone(); + minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"), + "value".to_string(), + )])); + + let (_, idx) = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(rustfs_alias.clone()), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(minio_alias), + idx: 1, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect("dynamic internal aliases with the same target should resolve"); + assert_eq!(idx, 1); + + let mut different_target_case = base; + different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([( + format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"), + "value".to_string(), + )])); + + assert_equal_time_identity_conflict(rustfs_alias, different_target_case); + } + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + let mut first = base.clone(); + first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([ + ("x-rustfs-internal-compression".to_string(), "zstd".to_string()), + ("x-minio-internal-compression".to_string(), "snappy".to_string()), + ])); + let mut second = base; + second.user_defined = first.user_defined.clone(); + + assert_equal_time_identity_conflict(first, second); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() { + let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())); + + let mut replication = base.clone(); + replication.replication_status_internal = Some("PENDING".to_string()); + replication.replication_status = ReplicationStatusType::Pending; + assert_equal_time_identity_conflict(base.clone(), replication); + + let mut purge = base.clone(); + purge.version_purge_status_internal = Some("PENDING".to_string()); + purge.version_purge_status = VersionPurgeStatusType::Pending; + assert_equal_time_identity_conflict(base.clone(), purge); + + let mut decision = base; + decision.replication_decision = "replicate".to_string(); + assert_equal_time_identity_conflict( + object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())), + decision, + ); + } + + #[test] + fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() { + let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string())); + without_mod_time.mod_time = None; + let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string())); + + assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch); + } + + #[test] + fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() { + let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string())); + let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string())); + older.data_dir = Some(Uuid::from_u128(2)); + + let (info, idx) = resolve_latest_object_info_candidates( + vec![ + LatestObjectInfoCandidate { + info: Some(latest), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: Some(older), + idx: 9, + err: None, + }, + ], + "bucket", + "object", + &ObjectOptions::default(), + ) + .expect("older identity divergence must not affect the latest candidate"); + + assert_eq!(idx, 0); + assert_eq!( + info.mod_time, + Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed")) + ); + } + + #[test] + fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() { + let candidates = vec![ + LatestObjectInfoCandidate { + info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))), + idx: 0, + err: None, + }, + LatestObjectInfoCandidate { + info: None, + idx: 1, + err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())), + }, + ]; + + let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default()) + .expect("not-found pools must not block resolution of found candidates"); + + assert_eq!(idx, 0); + assert_eq!(info.version_id, Some(Uuid::from_u128(1))); + } + #[test] fn resolve_latest_object_info_candidates_returns_non_not_found_error() { let err = resolve_latest_object_info_candidates( diff --git a/crates/ecstore/src/store/rebalance/support.rs b/crates/ecstore/src/store/rebalance/support.rs index 6035e58d8..6e4db41b8 100644 --- a/crates/ecstore/src/store/rebalance/support.rs +++ b/crates/ecstore/src/store/rebalance/support.rs @@ -12,10 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::Ordering; +use std::collections::HashMap; use crate::error::{Error, Result, StorageError, is_err_object_not_found, is_err_version_not_found}; use crate::object_api::{ObjectInfo, ObjectOptions}; +use rustfs_utils::http::metadata_compat::{ + SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, SUFFIX_REPLICATION_RESET_ARN_PREFIX, + strip_internal_prefix_preserving_case, +}; use rustfs_utils::path::decode_dir_object; use time::OffsetDateTime; @@ -73,7 +77,7 @@ pub(super) fn resolve_rebalance_delete_from_all_pools_result( object: &str, ) -> Result { result.map_err(|err| { - if err == Error::PreconditionFailed { + if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) { err } else { Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}")) @@ -86,7 +90,7 @@ fn is_ignorable_rebalance_delete_error(err: &Error) -> bool { } fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error { - if err == Error::PreconditionFailed { + if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) { err } else { Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}")) @@ -137,37 +141,158 @@ pub(super) fn rebalance_disk_set_lookup_error(pool_idx: usize, set_idx: usize, p )) } +fn latest_candidate_mod_time(candidate: &LatestObjectInfoCandidate) -> Option { + candidate + .info + .as_ref() + .map(|info| info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH)) +} + +fn same_transition_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool { + left.transition_version_state == right.transition_version_state + && left.transitioned_object.name == right.transitioned_object.name + && left.transitioned_object.version_id == right.transitioned_object.version_id + && left.transitioned_object.tier == right.transitioned_object.tier + && left.transitioned_object.free_version == right.transitioned_object.free_version + && left.transitioned_object.status == right.transitioned_object.status +} + +#[derive(PartialEq, Eq)] +struct LatestUserDefinedIdentity { + internal: HashMap, + other: HashMap, +} + +fn normalize_internal_identity_suffix(key: &str) -> Option { + let suffix = strip_internal_prefix_preserving_case(key)?; + + for dynamic_prefix in [ + SUFFIX_REPLICATION_RESET_ARN_PREFIX, + SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX, + ] { + let prefix_len = dynamic_prefix.len(); + if let (Some(prefix), Some(remainder)) = (suffix.get(..prefix_len), suffix.get(prefix_len..)) + && prefix.eq_ignore_ascii_case(dynamic_prefix) + { + return Some(format!("{dynamic_prefix}{remainder}")); + } + } + + Some(suffix.to_lowercase()) +} + +fn normalize_user_defined_identity(user_defined: &HashMap) -> Option { + let mut identity = LatestUserDefinedIdentity { + internal: HashMap::with_capacity(user_defined.len()), + other: HashMap::with_capacity(user_defined.len()), + }; + + for (key, value) in user_defined { + if let Some(suffix) = normalize_internal_identity_suffix(key) { + if identity + .internal + .insert(suffix, value.clone()) + .is_some_and(|previous| previous != *value) + { + return None; + } + } else { + identity.other.insert(key.clone(), value.clone()); + } + } + + Some(identity) +} + +fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool { + match ( + normalize_user_defined_identity(&left.user_defined), + normalize_user_defined_identity(&right.user_defined), + ) { + (Some(left), Some(right)) => left == right, + _ => false, + } +} + +/// Pool-specific erasure geometry is intentionally excluded: `get_object_info` +/// returns each pool's own `data_blocks`/`parity_blocks`, so those values can +/// differ for the same object version while the selected winner still carries +/// the chosen pool's layout. `put_object_reader` is also intentionally +/// excluded because it is a transient request handle that `ObjectInfo::clone` +/// drops. Every other ObjectInfo field is part of the production-visible +/// identity and must agree before the pool index can provide a deterministic +/// tie-break. +fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool { + left.bucket == right.bucket + && left.name == right.name + && left.storage_class == right.storage_class + && left.mod_time == right.mod_time + && left.size == right.size + && left.actual_size == right.actual_size + && left.is_dir == right.is_dir + && same_user_defined_identity(left, right) + && left.user_tags == right.user_tags + && left.version_id == right.version_id + && left.data_dir == right.data_dir + && left.delete_marker == right.delete_marker + && same_transition_identity(left, right) + && left.restore_ongoing == right.restore_ongoing + && left.restore_expires == right.restore_expires + && left.parts == right.parts + && left.is_latest == right.is_latest + && left.content_type == right.content_type + && left.content_encoding == right.content_encoding + && left.expires == right.expires + && left.num_versions == right.num_versions + && left.successor_mod_time == right.successor_mod_time + && left.etag == right.etag + && left.inlined == right.inlined + && left.metadata_only == right.metadata_only + && left.version_only == right.version_only + && left.replication_status_internal == right.replication_status_internal + && left.replication_status == right.replication_status + && left.version_purge_status_internal == right.version_purge_status_internal + && left.version_purge_status == right.version_purge_status + && left.replication_decision == right.replication_decision + && left.checksum == right.checksum +} + pub(super) fn resolve_latest_object_info_candidates( - mut candidates: Vec, + candidates: Vec, bucket: &str, object: &str, opts: &ObjectOptions, ) -> Result<(ObjectInfo, usize)> { - candidates.sort_by(|a, b| { - let a_mod = if let Some(info) = &a.info { - info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) - } else { - OffsetDateTime::UNIX_EPOCH + let latest_mod_time = candidates.iter().filter_map(latest_candidate_mod_time).max(); + + if let Some(latest_mod_time) = latest_mod_time { + let mut latest_candidates = candidates + .into_iter() + .filter(|candidate| latest_candidate_mod_time(candidate) == Some(latest_mod_time)) + .collect::>(); + + latest_candidates.sort_by(|left, right| right.idx.cmp(&left.idx)); + + let Some(winner) = latest_candidates.first() else { + return Err(Error::ErasureReadQuorum); + }; + let Some(winner_info) = winner.info.as_ref() else { + return Err(Error::ErasureReadQuorum); }; - let b_mod = if let Some(info) = &b.info { - info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) - } else { - OffsetDateTime::UNIX_EPOCH - }; - - if a_mod == b_mod { - return if a.idx < b.idx { Ordering::Greater } else { Ordering::Less }; + if latest_candidates.iter().skip(1).any(|candidate| { + candidate + .info + .as_ref() + .is_none_or(|info| !same_latest_object_info_identity(winner_info, info)) + }) { + return Err(Error::ErasureReadQuorum); } - b_mod.cmp(&a_mod) - }); + return Ok((winner_info.clone(), winner.idx)); + } for candidate in candidates { - if let Some(info) = candidate.info { - return Ok((info, candidate.idx)); - } - if let Some(err) = candidate.err && !is_err_object_not_found(&err) && !is_err_version_not_found(&err) @@ -191,6 +316,18 @@ mod tests { assert_eq!(err, Error::PreconditionFailed); } + #[test] + fn rebalance_delete_result_preserves_prefix_access_denied() { + let err = resolve_rebalance_delete_from_all_pools_result( + Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())), + "bucket", + "object", + ) + .expect_err("prefix access denial should remain structured"); + + assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())); + } + #[test] fn rebalance_delete_pool_result_preserves_precondition_failed() { let err = resolve_rebalance_delete_from_all_pools_results( @@ -205,4 +342,19 @@ mod tests { assert_eq!(err, Error::PreconditionFailed); } + + #[test] + fn rebalance_delete_pool_result_preserves_prefix_access_denied() { + let err = resolve_rebalance_delete_from_all_pools_results( + vec![RebalanceDeletePoolResult { + pool_idx: 0, + result: Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())), + }], + "bucket", + "object", + ) + .expect_err("prefix access denial should remain structured"); + + assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())); + } } diff --git a/crates/io-metrics/src/internode_metrics.rs b/crates/io-metrics/src/internode_metrics.rs index 65f93c2a1..6b1ea382c 100644 --- a/crates/io-metrics/src/internode_metrics.rs +++ b/crates/io-metrics/src/internode_metrics.rs @@ -54,6 +54,13 @@ pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "read_versio pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "read_version_response_msgpack_encode"; pub const INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP: &str = "read_version_rpc_roundtrip"; pub const INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE: &str = "read_version_response_decode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE: &str = "batch_read_version_request_encode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE: &str = "batch_read_version_request_decode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ: &str = "batch_read_version_disk_read"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE: &str = "batch_read_version_response_json_encode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE: &str = "batch_read_version_response_msgpack_encode"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP: &str = "batch_read_version_rpc_roundtrip"; +pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read_version_response_decode"; const OPERATION_LABEL: &str = "operation"; const BACKEND_LABEL: &str = "backend"; diff --git a/crates/madmin/src/metrics.rs b/crates/madmin/src/metrics.rs index 49e1e5ff2..4e55563a5 100644 --- a/crates/madmin/src/metrics.rs +++ b/crates/madmin/src/metrics.rs @@ -689,6 +689,14 @@ pub struct ScannerMetrics { pub cycle_max_objects: u64, #[serde(rename = "cycle_max_directories", default)] pub cycle_max_directories: u64, + #[serde(rename = "cycle_timeout_total", default)] + pub cycle_timeout_total: u64, + #[serde(rename = "cycle_recovery_required_total", default)] + pub cycle_recovery_required_total: u64, + #[serde(rename = "cycle_last_progress_age", default)] + pub cycle_last_progress_age: u64, + #[serde(rename = "leader_lease_without_progress", default)] + pub leader_lease_without_progress: bool, #[serde(rename = "bitrot_cycle_enabled", default)] pub bitrot_cycle_enabled: bool, #[serde(rename = "bitrot_cycle_seconds", default)] @@ -764,6 +772,8 @@ impl ScannerMetrics { self.cycle_max_duration_seconds = other.cycle_max_duration_seconds; self.cycle_max_objects = other.cycle_max_objects; self.cycle_max_directories = other.cycle_max_directories; + self.cycle_last_progress_age = other.cycle_last_progress_age; + self.leader_lease_without_progress = other.leader_lease_without_progress; self.bitrot_cycle_enabled = other.bitrot_cycle_enabled; self.bitrot_cycle_seconds = other.bitrot_cycle_seconds; } @@ -857,6 +867,12 @@ impl ScannerMetrics { .saturating_add(other.last_cycle_replication_checks); self.last_cycle_usage_saves = self.last_cycle_usage_saves.saturating_add(other.last_cycle_usage_saves); self.failed_cycles = self.failed_cycles.saturating_add(other.failed_cycles); + self.cycle_timeout_total = self.cycle_timeout_total.saturating_add(other.cycle_timeout_total); + self.cycle_recovery_required_total = self + .cycle_recovery_required_total + .saturating_add(other.cycle_recovery_required_total); + self.cycle_last_progress_age = self.cycle_last_progress_age.max(other.cycle_last_progress_age); + self.leader_lease_without_progress |= other.leader_lease_without_progress; self.superseded_cycles = self.superseded_cycles.saturating_add(other.superseded_cycles); self.partial_cycles_unknown = self.partial_cycles_unknown.saturating_add(other.partial_cycles_unknown); self.partial_cycles_runtime = self.partial_cycles_runtime.saturating_add(other.partial_cycles_runtime); diff --git a/crates/policy/src/policy/policy.rs b/crates/policy/src/policy/policy.rs index d412fa789..946f930a8 100644 --- a/crates/policy/src/policy/policy.rs +++ b/crates/policy/src/policy/policy.rs @@ -195,7 +195,8 @@ pub struct BucketPolicyArgs<'a> { #[derive(Serialize, Deserialize, Clone, Default, Debug)] #[serde(deny_unknown_fields)] pub struct BucketPolicy { - #[serde(default, rename = "Id", skip_serializing_if = "ID::is_empty")] + // RUSTFS_COMPAT_TODO(rustfs-6339): accept bucket policies persisted with the legacy "ID" key. Remove after migration tooling rewrites every retained legacy bucket policy. + #[serde(default, rename = "Id", alias = "ID", skip_serializing_if = "ID::is_empty")] pub id: ID, #[serde(rename = "Version")] pub version: String, @@ -2786,7 +2787,7 @@ mod test { let parsed: serde_json::Value = serde_json::from_str(&json).expect("Should parse"); // Verify empty fields are omitted - assert!(!parsed.as_object().unwrap().contains_key("ID"), "Empty ID should be omitted"); + assert!(parsed.get("Id").is_none(), "Empty ID should be omitted"); let statement = &parsed["Statement"][0]; assert!(!statement.as_object().unwrap().contains_key("Sid"), "Empty Sid should be omitted"); @@ -2809,6 +2810,43 @@ mod test { assert_eq!(statement["Principal"]["AWS"], "*"); } + #[test] + fn test_bucket_policy_deserializes_legacy_id() { + let legacy_policy = br#"{"ID":"","Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"NotAction":[],"Resource":["arn:aws:s3:::bucket/*"],"NotResource":[],"Condition":{}}]}"#; + + let policy: BucketPolicy = + serde_json::from_slice(legacy_policy).expect("bucket policy with legacy ID should deserialize"); + assert!(policy.id.is_empty()); + policy.is_valid().expect("legacy bucket policy should remain valid"); + + let policy: BucketPolicy = serde_json::from_str(r#"{"ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#) + .expect("non-empty legacy ID should deserialize"); + assert_eq!(policy.id.0, "legacy-policy"); + + let serialized = serde_json::to_value(&policy).expect("bucket policy should serialize"); + assert_eq!(serialized["Id"], "legacy-policy"); + assert!(serialized.get("ID").is_none(), "legacy ID spelling should not be serialized"); + } + + #[test] + fn test_bucket_policy_legacy_id_alias_remains_strict() { + let unknown_field = r#"{"Version":"2012-10-17","Statement":[],"Unexpected":true}"#; + let error = + serde_json::from_str::(unknown_field).expect_err("unrelated unknown fields should remain rejected"); + assert!( + error.to_string().contains("unknown field `Unexpected`"), + "unexpected deserialization error: {error}" + ); + + let duplicate_id = r#"{"Id":"current-policy","ID":"legacy-policy","Version":"2012-10-17","Statement":[]}"#; + let error = serde_json::from_str::(duplicate_id) + .expect_err("canonical and legacy ID fields should not be accepted together"); + assert!( + error.to_string().contains("duplicate field `Id`"), + "unexpected deserialization error: {error}" + ); + } + #[test] fn test_existing_object_tag_condition_helpers() { let identity_policy = Policy::parse_config( diff --git a/crates/scanner/src/runtime_config.rs b/crates/scanner/src/runtime_config.rs index 48fb40d2d..2fef6f9f4 100644 --- a/crates/scanner/src/runtime_config.rs +++ b/crates/scanner/src/runtime_config.rs @@ -125,7 +125,10 @@ impl Default for ScannerRuntimeConfig { cycle_interval_source: ScannerRuntimeConfigSource::Default, bitrot_cycle: Some(Duration::from_secs(DEFAULT_HEAL_BITROT_CYCLE_SECS)), bitrot_cycle_source: ScannerRuntimeConfigSource::Default, - cycle_budget: ScannerCycleBudgetConfig::default(), + cycle_budget: ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)), + ..Default::default() + }, cycle_max_duration_source: ScannerRuntimeConfigSource::Default, cycle_max_objects_source: ScannerRuntimeConfigSource::Default, cycle_max_directories_source: ScannerRuntimeConfigSource::Default, @@ -374,7 +377,10 @@ fn validate_persisted_scanner_runtime_config(config: &ServerConfig) -> Result<() } validate_optional_config_u64(scanner_kvs, SCANNER_START_DELAY, "")?; validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE, "")?; - validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)?; + if let Some(value) = config_value(scanner_kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) { + let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?; + cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs)?; + } validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_OBJECTS, DEFAULT_SCANNER_CYCLE_MAX_OBJECTS)?; validate_optional_config_u64(scanner_kvs, SCANNER_CYCLE_MAX_DIRECTORIES, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES)?; if let Some(value) = config_value(heal_kvs, HEAL_BITROT_CYCLE, DEFAULT_HEAL_BITROT_CYCLE_SECS) { @@ -436,19 +442,46 @@ fn lookup_max_wait( Ok((speed.max_sleep(), speed_source)) } -fn lookup_optional_seconds( - kvs: Option<&KVS>, - key: &'static str, - env_key: &'static str, - default: u64, -) -> Result<(Option, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> { - if let Some(secs) = rustfs_utils::get_env_opt_u64(env_key) { - return Ok((Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Env)); +fn lookup_cycle_duration(kvs: Option<&KVS>) -> Result<(Option, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> { + match rustfs_utils::get_env_parse_outcome::(ENV_SCANNER_CYCLE_MAX_DURATION_SECS) { + rustfs_utils::EnvParseOutcome::Parsed(secs) => { + return cycle_duration_from_secs(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, secs) + .map(|duration| (duration, ScannerRuntimeConfigSource::Env)); + } + rustfs_utils::EnvParseOutcome::Invalid => { + // Do not include the raw environment value in the typed error: + // deployments occasionally put sensitive material in inherited + // environment snapshots. The key still identifies the control. + return Err(invalid_value( + ENV_SCANNER_CYCLE_MAX_DURATION_SECS, + "", + "expected unsigned integer seconds", + )); + } + rustfs_utils::EnvParseOutcome::Absent => {} } - if let Some(value) = config_value(kvs, key, default) { - return parse_config_u64(key, value).map(|secs| (Some(Duration::from_secs(secs)), ScannerRuntimeConfigSource::Config)); + + if let Some(value) = config_value(kvs, SCANNER_CYCLE_MAX_DURATION, DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS) { + let secs = parse_config_u64(SCANNER_CYCLE_MAX_DURATION, value)?; + return cycle_duration_from_secs(SCANNER_CYCLE_MAX_DURATION, secs) + .map(|duration| (duration, ScannerRuntimeConfigSource::Config)); } - Ok((None, ScannerRuntimeConfigSource::Default)) + + Ok(( + Some(Duration::from_secs(DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS)), + ScannerRuntimeConfigSource::Default, + )) +} + +fn cycle_duration_from_secs(key: &'static str, secs: u64) -> Result, ScannerRuntimeConfigError> { + if secs == 0 { + return Ok(None); + } + let duration = Duration::from_secs(secs); + if std::time::Instant::now().checked_add(duration).is_none() { + return Err(invalid_value(key, "", "duration exceeds the timer range")); + } + Ok(Some(duration)) } fn lookup_start_delay(kvs: Option<&KVS>) -> Result<(Option, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> { @@ -553,12 +586,7 @@ pub(crate) fn lookup_scanner_runtime_config( (speed.cycle_interval(), speed_source) }; - let (cycle_max_duration, cycle_max_duration_source) = lookup_optional_seconds( - scanner_kvs, - SCANNER_CYCLE_MAX_DURATION, - ENV_SCANNER_CYCLE_MAX_DURATION_SECS, - DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS, - )?; + let (cycle_max_duration, cycle_max_duration_source) = lookup_cycle_duration(scanner_kvs)?; let (cycle_max_objects, cycle_max_objects_source) = lookup_count_budget( scanner_kvs, SCANNER_CYCLE_MAX_OBJECTS, @@ -863,10 +891,10 @@ mod tests { use rustfs_config::server_config::{Config as ServerConfig, KVS}; use rustfs_config::{ DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, - ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, - HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, - SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, - SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed, + ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, + ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, + SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, + SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed, }; use std::collections::HashMap; use std::time::Duration; @@ -941,6 +969,50 @@ mod tests { }); } + #[test] + fn scanner_unset_budget_uses_safe_default_but_explicit_zero_is_unbounded() { + let config = server_config_with_scanner(&[]); + with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || { + let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config"); + assert_eq!(resolved.cycle_budget.max_duration, Some(Duration::from_secs(1800))); + assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Default); + }); + + let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "0")]); + with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || { + let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config"); + assert_eq!(resolved.cycle_budget.max_duration, None); + assert_eq!(resolved.cycle_max_duration_source, ScannerRuntimeConfigSource::Config); + }); + } + + #[test] + fn cycle_budget_invalid_or_overflow_config_is_rejected() { + with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("invalid"), || { + let error = lookup_scanner_runtime_config(None).expect_err("invalid duration env must be rejected"); + assert!(error.to_string().contains(ENV_SCANNER_CYCLE_MAX_DURATION_SECS)); + assert!(error.to_string().contains("")); + assert!(!error.to_string().contains(": invalid (")); + }); + with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551616"), || { + assert!(lookup_scanner_runtime_config(None).is_err()); + }); + with_var(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, Some("18446744073709551615"), || { + assert!(lookup_scanner_runtime_config(None).is_err()); + }); + let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "not-a-duration")]); + assert!(lookup_scanner_runtime_config(Some(&config)).is_err()); + } + + #[test] + fn scanner_runtime_config_validation_rejects_overflow_persisted_duration() { + let config = server_config_with_scanner(&[(SCANNER_CYCLE_MAX_DURATION, "18446744073709551615")]); + + let error = validate_scanner_runtime_config(&config) + .expect_err("persisted duration that exceeds the timer range must be rejected"); + assert!(error.to_string().contains(SCANNER_CYCLE_MAX_DURATION)); + } + #[test] fn scanner_runtime_config_normalizes_persisted_default_speed() { let config = server_config_with_scanner(&[(SCANNER_SPEED, "default")]); diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 7a4a482cf..059de1fb4 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -52,6 +52,7 @@ use rustfs_config::{ }; use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS}; use rustfs_data_usage::observed_data_usage_is_newer; +use rustfs_lock::NamespaceLockGuard; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; use tokio::sync::{Notify, mpsc}; @@ -1037,20 +1038,116 @@ fn data_usage_persist_timeout() -> Duration { DataUsageCache::persistence_timeout() } +#[cfg(not(test))] +const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_secs(30); +#[cfg(test)] +const SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT: Duration = Duration::from_millis(50); + +async fn fence_scanner_epoch_after_cycle_timeout( + ctx: &CancellationToken, + storeapi: Arc, + cycle_info: &mut CurrentCycle, + cycle_revision: &mut DataUsageCacheRevision, + leader_epoch: &mut u64, + lock_lost: LockLost, +) -> bool +where + Store: ScannerObjectIO, + LockLost: Future, +{ + let fence_ctx = ctx.child_token(); + let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch); + tokio::pin!(claim); + tokio::pin!(lock_lost); + tokio::select! { + biased; + _ = &mut lock_lost => { + fence_ctx.cancel(); + false + } + result = tokio::time::timeout(SCANNER_CYCLE_EPOCH_FENCE_TIMEOUT, &mut claim) => { + result.unwrap_or(false) && !fence_ctx.is_cancelled() + } + } +} + +struct ScannerCycleDeadlineState<'a> { + cycle_info: &'a mut CurrentCycle, + cycle_revision: &'a mut DataUsageCacheRevision, + leader_epoch: &'a mut u64, + cycle_budget: &'a ScannerCycleBudget, +} + +fn cycle_timeout_requires_recovery(worker_stopped: bool, cycle_state_persisted: bool, generation_fenced: bool) -> bool { + !worker_stopped || !cycle_state_persisted || !generation_fenced +} + +async fn handle_scanner_cycle_deadline( + ctx: &CancellationToken, + storeapi: Arc, + state: ScannerCycleDeadlineState<'_>, + worker_stopped: bool, + guard: &mut NamespaceLockGuard, +) where + Store: ScannerObjectIO, +{ + let fenced = fence_scanner_epoch_after_cycle_timeout( + ctx, + storeapi, + state.cycle_info, + state.cycle_revision, + state.leader_epoch, + guard.lock_lost_notified(), + ) + .await; + let cycle_state_persisted = state.cycle_budget.cycle_state_persisted(); + let recovery_required = cycle_timeout_requires_recovery(worker_stopped, cycle_state_persisted, fenced); + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_CYCLE_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "cycle_timeout", + worker_stopped, + cycle_state_persisted, + generation_fenced = fenced, + recovery_required, + "Scanner cycle deadline expired; durable cursor/generation fencing completed when possible" + ); + global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age()); + // Stop renewing before releasing the lease. A new leader can then claim the + // higher persisted generation instead of inheriting the expired worker. + guard.release(); + global_metrics().set_cycle(None).await; +} + async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) { cycle_info.current = 0; global_metrics().clear_current_scan_mode(); cycle_metrics_guard.finish(cycle_info.clone()).await; } -#[instrument(skip_all)] -#[hotpath::measure] +#[cfg(test)] async fn run_data_scanner_cycle( ctx: &CancellationToken, storeapi: &Arc, cycle_info: &mut CurrentCycle, cycle_revision: &mut DataUsageCacheRevision, leader_epoch: u64, +) -> ScannerCycleOutcome { + let cycle_budget = ScannerCycleBudget::new(ctx, scanner_cycle_budget_config()); + run_data_scanner_cycle_with_budget(ctx, storeapi, cycle_info, cycle_revision, leader_epoch, cycle_budget).await +} + +#[instrument(skip_all)] +#[hotpath::measure] +async fn run_data_scanner_cycle_with_budget( + ctx: &CancellationToken, + storeapi: &Arc, + cycle_info: &mut CurrentCycle, + cycle_revision: &mut DataUsageCacheRevision, + leader_epoch: u64, + cycle_budget: Arc, ) -> ScannerCycleOutcome { let _activity_guard = ScannerActivityGuard::new(); if let Err(err) = refresh_scanner_runtime_config_from_global() { @@ -1066,7 +1163,11 @@ async fn run_data_scanner_cycle( } let configured_cycle_interval = scanner_cycle_interval(); let configured_bitrot_cycle = scanner_bitrot_cycle(); - let cycle_budget_config = scanner_cycle_budget_config(); + let cycle_budget_config = ScannerCycleBudgetConfig { + max_duration: cycle_budget.max_duration(), + max_objects: cycle_budget.max_objects(), + max_directories: cycle_budget.max_directories(), + }; let usage_persist_timeout = data_usage_persist_timeout(); global_metrics().record_scanner_cycle_config( configured_cycle_interval, @@ -1137,7 +1238,6 @@ async fn run_data_scanner_cycle( let (sender, receiver) = mpsc::channel::(1); let done_cycle = Metrics::time(Metric::ScanCycle); - let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config); let scan_result = storeapi .clone() .nsscanner_with_status( @@ -1277,7 +1377,7 @@ async fn run_data_scanner_cycle( "Scanner cycle is recovering to a newer durable cache generation" ); emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); - return if persist_required_scanner_cycle_floor( + let persisted = persist_required_scanner_cycle_floor( ctx, storeapi.clone(), cycle_info, @@ -1286,8 +1386,9 @@ async fn run_data_scanner_cycle( required_cycle, &mut cycle_metrics_guard, ) - .await - { + .await; + return if persisted { + cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial } else { ScannerCycleOutcome::Failed @@ -1345,7 +1446,7 @@ async fn run_data_scanner_cycle( scan_cycle_partial_reason(budget_reason), scan_cycle_partial_source(budget_reason), ); - return if finalize_partial_scan_cycle( + let persisted = finalize_partial_scan_cycle( ctx, storeapi.clone(), cycle_info, @@ -1353,8 +1454,9 @@ async fn run_data_scanner_cycle( leader_epoch, &mut cycle_metrics_guard, ) - .await - { + .await; + return if persisted { + cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial } else { ScannerCycleOutcome::Failed @@ -1429,7 +1531,7 @@ async fn run_data_scanner_cycle( ); } emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None); - return if finalize_partial_scan_cycle( + let persisted = finalize_partial_scan_cycle( ctx, storeapi.clone(), cycle_info, @@ -1437,8 +1539,9 @@ async fn run_data_scanner_cycle( leader_epoch, &mut cycle_metrics_guard, ) - .await - { + .await; + return if persisted { + cycle_budget.mark_cycle_state_persisted(); ScannerCycleOutcome::Partial } else { ScannerCycleOutcome::Failed @@ -1479,6 +1582,7 @@ async fn run_data_scanner_cycle( ) .await { + cycle_budget.mark_cycle_state_persisted(); emit_scan_cycle_superseded(cycle_start.elapsed()); return ScannerCycleOutcome::Superseded; } @@ -1511,6 +1615,7 @@ async fn run_data_scanner_cycle( emit_scan_cycle_complete(false, cycle_start.elapsed()); return ScannerCycleOutcome::Failed; } + cycle_budget.mark_cycle_state_persisted(); done_cycle(); emit_scan_cycle_complete(true, cycle_start.elapsed()); @@ -1575,7 +1680,7 @@ async fn run_data_scanner_with_maintenance_state( ) -> Result<(), ScannerError> { reset_scanner_cycle_schedule(); // Acquire leader lock (write lock) to ensure only one scanner runs - let guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await { + let mut guard = match storeapi.new_ns_lock(RUSTFS_META_BUCKET, "leader.lock").await { Ok(ns_lock) => match ns_lock.get_write_lock_quiet(get_lock_acquire_timeout()).await { Ok(guard) => { record_scanner_leader_lock_state("acquired"); @@ -1740,13 +1845,49 @@ async fn run_data_scanner_with_maintenance_state( return Ok(()); } let cycle_ctx = ctx.child_token(); - let initial_outcome = await_scanner_cycle_with_lock_fence( + let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config()); + let initial_outcome = match await_scanner_cycle_with_budget_fence( &cycle_ctx, - run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch), + &cycle_budget, + run_data_scanner_cycle_with_budget( + &cycle_ctx, + &storeapi, + &mut cycle_info, + &mut cycle_revision, + leader_epoch, + cycle_budget.clone(), + ), guard.lock_lost_notified(), ) .await - .unwrap_or(ScannerCycleOutcome::Failed); + { + ScannerCycleWaitOutcome::Completed(outcome) => outcome, + ScannerCycleWaitOutcome::LockLost => { + record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await; + global_metrics().set_cycle(None).await; + return Ok(()); + } + ScannerCycleWaitOutcome::Cancelled => { + global_metrics().set_cycle(None).await; + return Ok(()); + } + ScannerCycleWaitOutcome::Deadline { worker_stopped } => { + handle_scanner_cycle_deadline( + &ctx, + storeapi.clone(), + ScannerCycleDeadlineState { + cycle_info: &mut cycle_info, + cycle_revision: &mut cycle_revision, + leader_epoch: &mut leader_epoch, + cycle_budget: &cycle_budget, + }, + worker_stopped, + &mut guard, + ) + .await; + return Ok(()); + } + }; superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded); deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_))); dirty_usage_generation_seen = dirty_generation_before_cycle; @@ -1952,13 +2093,49 @@ async fn run_data_scanner_with_maintenance_state( } let dirty_generation_before_cycle = dirty_usage_generation(); let cycle_ctx = ctx.child_token(); - let outcome = await_scanner_cycle_with_lock_fence( + let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config()); + let outcome = match await_scanner_cycle_with_budget_fence( &cycle_ctx, - run_data_scanner_cycle(&cycle_ctx, &storeapi, &mut cycle_info, &mut cycle_revision, leader_epoch), + &cycle_budget, + run_data_scanner_cycle_with_budget( + &cycle_ctx, + &storeapi, + &mut cycle_info, + &mut cycle_revision, + leader_epoch, + cycle_budget.clone(), + ), guard.lock_lost_notified(), ) .await - .unwrap_or(ScannerCycleOutcome::Failed); + { + ScannerCycleWaitOutcome::Completed(outcome) => outcome, + ScannerCycleWaitOutcome::LockLost => { + record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await; + global_metrics().set_cycle(None).await; + return Ok(()); + } + ScannerCycleWaitOutcome::Cancelled => { + global_metrics().set_cycle(None).await; + return Ok(()); + } + ScannerCycleWaitOutcome::Deadline { worker_stopped } => { + handle_scanner_cycle_deadline( + &ctx, + storeapi.clone(), + ScannerCycleDeadlineState { + cycle_info: &mut cycle_info, + cycle_revision: &mut cycle_revision, + leader_epoch: &mut leader_epoch, + cycle_budget: &cycle_budget, + }, + worker_stopped, + &mut guard, + ) + .await; + return Ok(()); + } + }; superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded); deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_))); dirty_usage_generation_seen = dirty_generation_before_cycle; diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index 32c893fdf..6459ae6e5 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -1581,3 +1581,63 @@ where output = &mut cycle => Some(output), } } + +#[derive(Debug, PartialEq, Eq)] +pub(super) enum ScannerCycleWaitOutcome { + Completed(T), + LockLost, + Cancelled, + Deadline { worker_stopped: bool }, +} + +pub(super) async fn await_scanner_cycle_with_budget_fence( + cycle_ctx: &CancellationToken, + budget: &ScannerCycleBudget, + cycle: Cycle, + lock_lost: LockLost, +) -> ScannerCycleWaitOutcome +where + Cycle: Future, + LockLost: Future, +{ + tokio::pin!(cycle); + tokio::pin!(lock_lost); + let deadline = async { + if let Some(deadline) = budget.deadline() { + tokio::time::sleep_until(deadline).await; + } else { + std::future::pending::<()>().await; + } + }; + tokio::pin!(deadline); + tokio::select! { + biased; + _ = &mut lock_lost => { + cycle_ctx.cancel(); + let _ = tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await; + ScannerCycleWaitOutcome::LockLost + } + _ = &mut deadline => { + budget.cancel_for_runtime(); + // Let the budget cancellation reach the scanner first so it can + // persist a partial cursor. Only an uncooperative worker gets the + // parent cancellation, and it is dropped after the bounded window; + // the caller fences its epoch next. + let worker_stopped = if tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle) + .await + .is_ok() + { + true + } else { + cycle_ctx.cancel(); + false + }; + ScannerCycleWaitOutcome::Deadline { worker_stopped } + } + _ = cycle_ctx.cancelled() => { + let _ = tokio::time::timeout(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, &mut cycle).await; + ScannerCycleWaitOutcome::Cancelled + } + output = &mut cycle => ScannerCycleWaitOutcome::Completed(output), + } +} diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 5430e9d29..d73d63cde 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -26,6 +26,7 @@ use std::task::Poll; use temp_env::{with_var, with_var_unset}; use tokio::io::AsyncReadExt; use tokio::sync::Mutex; +use tokio::time::{Duration, advance}; const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60; @@ -118,6 +119,178 @@ async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() { assert!(cycle_ctx.is_cancelled()); } +#[tokio::test(start_paused = true)] +async fn cycle_budget_fences_late_writer_after_timeout() { + let cycle_ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &cycle_ctx, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(5)), + ..Default::default() + }, + ); + let outcome = { + let cycle = std::future::pending::<()>(); + let lock_lost = std::future::pending::<()>(); + let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, cycle, lock_lost); + tokio::pin!(waiter); + tokio::task::yield_now().await; + advance(Duration::from_secs(5)).await; + tokio::task::yield_now().await; + advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await; + waiter.await + }; + assert_eq!(outcome, ScannerCycleWaitOutcome::Deadline { worker_stopped: false }); + assert!(cycle_ctx.is_cancelled()); + assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Runtime)); + + // A newer leadership epoch is the durable fence that rejects a late + // writer after the timed-out future has been dropped. + let store = Arc::new(MemoryConfigStore::default()); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 0, + next: 12, + ..Default::default() + }; + let persist_ctx = CancellationToken::new(); + assert!(persist_scanner_cycle_state(&persist_ctx, store.clone(), &mut cycle, &mut revision, 1).await); + let newer = encode_scanner_cycle_state(&cycle, 2).expect("new epoch fence should encode"); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.interleaving_puts.lock().await.insert(key, (2, newer)); + let mut late_cycle = CurrentCycle { next: 13, ..cycle }; + assert!(!persist_scanner_cycle_state(&persist_ctx, store, &mut late_cycle, &mut revision, 1).await); +} + +#[tokio::test(start_paused = true)] +async fn cycle_budget_parent_cancellation_is_not_reported_as_timeout() { + let cycle_ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &cycle_ctx, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(5)), + ..Default::default() + }, + ); + let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, std::future::pending::<()>(), std::future::pending()); + tokio::pin!(waiter); + tokio::task::yield_now().await; + cycle_ctx.cancel(); + tokio::task::yield_now().await; + advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await; + assert_eq!(waiter.await, ScannerCycleWaitOutcome::Cancelled); +} + +#[tokio::test(start_paused = true)] +async fn cycle_budget_deadline_wins_same_tick_as_parent_cancellation() { + let cycle_ctx = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &cycle_ctx, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(5)), + ..Default::default() + }, + ); + let waiter = await_scanner_cycle_with_budget_fence(&cycle_ctx, &budget, std::future::pending::<()>(), std::future::pending()); + tokio::pin!(waiter); + tokio::task::yield_now().await; + advance(Duration::from_secs(5)).await; + cycle_ctx.cancel(); + tokio::task::yield_now().await; + advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT).await; + + assert_eq!(waiter.await, ScannerCycleWaitOutcome::Deadline { worker_stopped: false }); + assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Runtime)); +} + +#[tokio::test] +async fn cycle_budget_persist_cursor_failure_is_recovery_required() { + let store = Arc::new(MemoryConfigStore::default()); + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + store.fail_put_number.lock().await.insert(key, 1); + + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + current: 12, + next: 12, + ..Default::default() + }; + let mut leader_epoch = 1; + let fenced = fence_scanner_epoch_after_cycle_timeout( + &ctx, + store, + &mut cycle, + &mut revision, + &mut leader_epoch, + std::future::pending(), + ) + .await; + assert!(!fenced, "a failed cursor/generation write must require recovery"); + let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()); + assert!(cycle_timeout_requires_recovery(true, budget.cycle_state_persisted(), fenced)); + + let metrics = Metrics::new(); + metrics.record_scanner_cycle_timeout(!fenced, Duration::from_secs(17)); + let report = metrics.report().await; + assert_eq!(report.cycle_timeout_total, 1); + assert_eq!(report.cycle_recovery_required_total, 1); + assert_eq!(report.cycle_last_progress_age, 17); + assert!(report.leader_lease_without_progress); +} + +#[tokio::test] +async fn cycle_budget_deadline_handler_fences_and_releases_guard() { + let (_temp_dir, store) = setup_scanner_cycle_store().await; + let lock = store + .new_ns_lock(RUSTFS_META_BUCKET, "leader.lock") + .await + .expect("scanner leader lock should be created"); + let mut guard = lock + .get_write_lock(Duration::from_secs(1)) + .await + .expect("scanner leader lock should be acquired"); + + let ctx = CancellationToken::new(); + let mut cycle_info = CurrentCycle { + current: 12, + next: 12, + ..Default::default() + }; + let mut cycle_revision = DataUsageCacheRevision::Missing; + let mut leader_epoch = 1; + let budget = ScannerCycleBudget::new( + &ctx, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(60)), + ..Default::default() + }, + ); + budget.mark_cycle_state_persisted(); + + handle_scanner_cycle_deadline( + &ctx, + store.clone(), + ScannerCycleDeadlineState { + cycle_info: &mut cycle_info, + cycle_revision: &mut cycle_revision, + leader_epoch: &mut leader_epoch, + cycle_budget: &budget, + }, + true, + &mut guard, + ) + .await; + + assert!(guard.is_released()); + let persisted = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + .await + .expect("deadline handler should persist a fenced cursor"); + let (_, persisted_epoch) = decode_scanner_cycle_state(&persisted).expect("fenced cursor should decode"); + assert_eq!(persisted_epoch, 2); + global_metrics().set_cycle(None).await; +} + #[tokio::test] async fn scanner_cycle_recovery_wake_survives_wait_registration_race() { notify_scanner_cycle_recovery_wake(); @@ -428,13 +601,6 @@ fn test_scanner_cycle_max_duration_uses_env() { }); } -#[test] -fn test_scanner_cycle_max_duration_default_is_disabled() { - with_var_unset(ENV_SCANNER_CYCLE_MAX_DURATION_SECS, || { - assert_eq!(scanner_cycle_max_duration(), None); - }); -} - #[tokio::test] async fn test_scanner_cycle_budget_cancels_after_duration() { let parent = CancellationToken::new(); @@ -2242,7 +2408,7 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() { } #[tokio::test] -async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() { +async fn cycle_budget_lease_takeover_rejects_old_generation() { let store = Arc::new(MemoryConfigStore::default()); let ctx = CancellationToken::new(); let mut revision = DataUsageCacheRevision::Missing; @@ -2287,12 +2453,17 @@ async fn test_successful_old_epoch_commit_is_fenced_after_cancellation() { .await ); - let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) + let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) .await .expect("replacement leadership claim should persist"); let (claimed_cycle, claimed_epoch) = decode_scanner_cycle_state(&state).expect("replacement cycle state should decode"); assert_eq!(claimed_cycle.next, 14); assert_eq!(claimed_epoch, 2); + + let mut stale_cycle = CurrentCycle { next: 15, ..cycle }; + let mut stale_revision = DataUsageCacheRevision::Etag("memory-2".to_string()); + let stale_ctx = CancellationToken::new(); + assert!(!persist_scanner_cycle_state(&stale_ctx, store, &mut stale_cycle, &mut stale_revision, 1,).await); } #[tokio::test] diff --git a/crates/scanner/src/scanner_budget.rs b/crates/scanner/src/scanner_budget.rs index 43ce732d5..65743439f 100644 --- a/crates/scanner/src/scanner_budget.rs +++ b/crates/scanner/src/scanner_budget.rs @@ -14,17 +14,16 @@ use std::sync::{ Arc, - atomic::{AtomicU8, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering}, }; -use std::time::Instant; - -use tokio::time::Duration; +use tokio::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; const BUDGET_REASON_NONE: u8 = 0; const BUDGET_REASON_RUNTIME: u8 = 1; const BUDGET_REASON_OBJECTS: u8 = 2; const BUDGET_REASON_DIRECTORIES: u8 = 3; +const PROGRESS_CLOCK_SAMPLE_INTERVAL: u64 = 128; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub(crate) struct ScannerCycleBudgetConfig { @@ -63,29 +62,51 @@ pub struct ScannerCycleBudget { token: CancellationToken, reason: Arc, started_at: Instant, + deadline: Option, max_duration: Option, max_objects: Option, max_directories: Option, track_progress: bool, + track_unbounded_counts: bool, objects_scanned: AtomicU64, directories_started: AtomicU64, entries_visited: AtomicU64, + last_progress_millis: AtomicU64, + cycle_state_persisted: AtomicBool, } impl ScannerCycleBudget { + #[cfg(test)] pub(crate) fn new(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc { - Self::new_inner(parent, config, false) + Self::new_inner(parent, config, false, false) } pub(crate) fn new_with_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc { - Self::new_inner(parent, config, true) + Self::new_inner(parent, config, true, true) } - fn new_inner(parent: &CancellationToken, config: ScannerCycleBudgetConfig, track_progress: bool) -> Arc { + pub(crate) fn new_with_runtime_progress_tracking(parent: &CancellationToken, config: ScannerCycleBudgetConfig) -> Arc { + let track_progress = config.max_duration.is_some(); + Self::new_inner(parent, config, track_progress, false) + } + + fn new_inner( + parent: &CancellationToken, + config: ScannerCycleBudgetConfig, + track_progress: bool, + track_unbounded_counts: bool, + ) -> Arc { let token = parent.child_token(); let reason = Arc::new(AtomicU8::new(BUDGET_REASON_NONE)); + let started_at = Instant::now(); + let deadline = config.max_duration.map(|duration| match started_at.checked_add(duration) { + Some(deadline) => deadline, + // Runtime config rejects this range, but keep programmatic callers + // fail-closed instead of panicking or silently disabling the wall clock. + None => started_at, + }); - if let Some(duration) = config.max_duration { + if let Some(deadline) = deadline { let parent = parent.clone(); let token_wait = token.clone(); let token_cancel = token.clone(); @@ -94,7 +115,7 @@ impl ScannerCycleBudget { tokio::select! { _ = parent.cancelled() => {} _ = token_wait.cancelled() => {} - _ = tokio::time::sleep(duration) => { + _ = tokio::time::sleep_until(deadline) => { Self::cancel_for_reason(&reason, &token_cancel, ScannerCycleBudgetReason::Runtime); } } @@ -104,14 +125,18 @@ impl ScannerCycleBudget { Arc::new(Self { token, reason, - started_at: Instant::now(), + started_at, + deadline, max_duration: config.max_duration, max_objects: config.max_objects, max_directories: config.max_directories, track_progress, + track_unbounded_counts, objects_scanned: AtomicU64::new(0), directories_started: AtomicU64::new(0), entries_visited: AtomicU64::new(0), + last_progress_millis: AtomicU64::new(0), + cycle_state_persisted: AtomicBool::new(false), }) } @@ -131,6 +156,14 @@ impl ScannerCycleBudget { self.max_duration } + pub(crate) fn deadline(&self) -> Option { + self.deadline + } + + pub(crate) fn cancel_for_runtime(&self) { + self.cancel_for(ScannerCycleBudgetReason::Runtime); + } + pub(crate) fn max_objects(&self) -> Option { self.max_objects } @@ -173,15 +206,43 @@ impl ScannerCycleBudget { self.entries_visited.load(Ordering::Relaxed) } + pub(crate) fn mark_cycle_state_persisted(&self) { + self.cycle_state_persisted.store(true, Ordering::Release); + } + + pub(crate) fn cycle_state_persisted(&self) -> bool { + self.cycle_state_persisted.load(Ordering::Acquire) + } + + pub(crate) fn progress_age(&self) -> Duration { + let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX); + let last_progress = self.last_progress_millis.load(Ordering::Relaxed); + Duration::from_millis(elapsed_millis.saturating_sub(last_progress)) + } + + fn record_progress_sample(&self, event: u64) { + // Clock reads are sampled at batch/count boundaries; the scanner's + // per-object path does not add a second progress atomic. + if event == 0 || (event != 1 && !event.is_multiple_of(PROGRESS_CLOCK_SAMPLE_INTERVAL)) { + return; + } + let elapsed_millis = u64::try_from(self.started_at.elapsed().as_millis()).unwrap_or(u64::MAX); + self.last_progress_millis.store(elapsed_millis, Ordering::Relaxed); + } + pub(crate) fn record_entries_visited(&self, entries_visited: u64) { if self.track_progress { - saturating_fetch_add(&self.entries_visited, entries_visited); + let entries = saturating_fetch_add(&self.entries_visited, entries_visited); + self.record_progress_sample(entries); } } pub(crate) fn record_remote_progress(&self, objects_scanned: u64, directories_started: u64) { if self.track_progress || self.max_objects.is_some() { let objects = saturating_fetch_add(&self.objects_scanned, objects_scanned); + if self.track_progress { + self.record_progress_sample(objects); + } if self.max_objects.is_some_and(|max_objects| objects >= max_objects) { self.cancel_for(ScannerCycleBudgetReason::Objects); } @@ -189,9 +250,12 @@ impl ScannerCycleBudget { if self.track_progress || self.max_directories.is_some() { let directories = saturating_fetch_add(&self.directories_started, directories_started); + if self.track_progress { + self.record_progress_sample(directories); + } if self .max_directories - .is_some_and(|max_directories| directories > max_directories) + .is_some_and(|max_directories| directory_budget_exhausted(directories, max_directories)) { self.cancel_for(ScannerCycleBudgetReason::Directories); } @@ -207,14 +271,17 @@ impl ScannerCycleBudget { } pub(crate) fn try_start_directory(&self) -> bool { - if !self.track_progress && self.max_directories.is_none() { + if self.max_directories.is_none() && !self.track_unbounded_counts { return true; } let directories = saturating_fetch_add(&self.directories_started, 1); + if self.track_progress { + self.record_progress_sample(directories); + } if self .max_directories - .is_some_and(|max_directories| directories > max_directories) + .is_some_and(|max_directories| directory_budget_exhausted(directories, max_directories)) { self.cancel_for(ScannerCycleBudgetReason::Directories); return false; @@ -224,11 +291,14 @@ impl ScannerCycleBudget { } pub(crate) fn record_object_scanned(&self) { - if !self.track_progress && self.max_objects.is_none() { + if self.max_objects.is_none() && !self.track_unbounded_counts { return; } let objects = saturating_fetch_add(&self.objects_scanned, 1); + if self.track_progress { + self.record_progress_sample(objects); + } if self.max_objects.is_some_and(|max_objects| objects >= max_objects) { self.cancel_for(ScannerCycleBudgetReason::Objects); } @@ -259,6 +329,13 @@ fn saturating_fetch_add(value: &AtomicU64, delta: u64) -> u64 { } } +fn directory_budget_exhausted(directories: u64, max_directories: u64) -> bool { + // Saturation hides a remote max+1 update when the configured limit is the + // largest representable counter. Treat that boundary as exhausted rather + // than allowing work to continue indefinitely. + directories > max_directories || (directories == u64::MAX && max_directories == u64::MAX) +} + impl Drop for ScannerCycleBudget { fn drop(&mut self) { self.token.cancel(); @@ -401,6 +478,35 @@ mod tests { assert_eq!(directory_budget.reason(), Some(ScannerCycleBudgetReason::Directories)); } + #[test] + fn directory_budget_fails_closed_when_progress_saturates() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_directories: Some(u64::MAX), + ..Default::default() + }, + ); + + budget.record_remote_progress(0, u64::MAX); + + assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Directories)); + assert!(budget.token().is_cancelled()); + + let local_budget = ScannerCycleBudget::new( + &parent, + ScannerCycleBudgetConfig { + max_directories: Some(u64::MAX), + ..Default::default() + }, + ); + local_budget.record_remote_progress(0, u64::MAX - 1); + assert!(!local_budget.budget_elapsed()); + assert!(!local_budget.try_start_directory()); + assert_eq!(local_budget.reason(), Some(ScannerCycleBudgetReason::Directories)); + } + #[test] fn explicit_progress_tracking_counts_unbounded_remote_work_without_cancelling() { let parent = CancellationToken::new(); @@ -461,4 +567,29 @@ mod tests { assert!(object_limited.requires_serial_progress_accounting()); assert!(directory_limited.requires_serial_progress_accounting()); } + + #[tokio::test(start_paused = true)] + async fn progress_age_uses_virtual_time_and_sampled_progress() { + let parent = CancellationToken::new(); + let budget = ScannerCycleBudget::new_with_runtime_progress_tracking( + &parent, + ScannerCycleBudgetConfig { + max_duration: Some(Duration::from_secs(60)), + ..Default::default() + }, + ); + + tokio::time::advance(Duration::from_secs(5)).await; + assert_eq!(budget.progress_age(), Duration::from_secs(5)); + budget.record_entries_visited(1); + assert_eq!(budget.progress_age(), Duration::ZERO); + + tokio::time::advance(Duration::from_secs(2)).await; + for _ in 0..126 { + budget.record_entries_visited(1); + } + assert_eq!(budget.progress_age(), Duration::from_secs(2)); + budget.record_entries_visited(1); + assert_eq!(budget.progress_age(), Duration::ZERO); + } } diff --git a/crates/scanner/src/scanner_io.rs b/crates/scanner/src/scanner_io.rs index 677719a69..c104fb263 100644 --- a/crates/scanner/src/scanner_io.rs +++ b/crates/scanner/src/scanner_io.rs @@ -48,6 +48,7 @@ use time::OffsetDateTime; use tokio::sync::{Mutex, Notify, Semaphore, mpsc}; use tokio::time::Duration; use tokio_util::sync::CancellationToken; +use tokio_util::task::AbortOnDropHandle; use tracing::{debug, error, warn}; use crate::ScannerObjectInfo as ObjectInfo; diff --git a/crates/scanner/src/scanner_io/io_cache.rs b/crates/scanner/src/scanner_io/io_cache.rs index 3151b4c0c..aa0339e4b 100644 --- a/crates/scanner/src/scanner_io/io_cache.rs +++ b/crates/scanner/src/scanner_io/io_cache.rs @@ -314,7 +314,7 @@ impl ScannerIOCache for SetDisks { let ctx_clone = ctx.clone(); let completed_bucket_count = Arc::new(AtomicUsize::new(0)); let completed_bucket_count_clone = completed_bucket_count.clone(); - let collect_bucket_results_fut = tokio::spawn(async move { + let collect_bucket_results_fut = AbortOnDropHandle::new(tokio::spawn(async move { let mut cancelled = false; loop { @@ -333,7 +333,7 @@ impl ScannerIOCache for SetDisks { } } } - }); + })); let mut futs = Vec::new(); @@ -365,7 +365,7 @@ impl ScannerIOCache for SetDisks { NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch), NamespaceScannerWorkerMode::Coordinator => None, }; - futs.push(tokio::spawn(async move { + futs.push(AbortOnDropHandle::new(tokio::spawn(async move { let remote_session_id = uuid::Uuid::new_v4(); let mut remote_session_sequence = 0_u64; loop { @@ -1038,7 +1038,7 @@ impl ScannerIOCache for SetDisks { ); } } - })); + }))); } drop(bucket_tx); drop(bucket_result_tx); diff --git a/crates/scanner/src/scanner_io/io_cycle.rs b/crates/scanner/src/scanner_io/io_cycle.rs index c2a8eb253..64763655f 100644 --- a/crates/scanner/src/scanner_io/io_cycle.rs +++ b/crates/scanner/src/scanner_io/io_cycle.rs @@ -242,7 +242,7 @@ impl ScannerIOCycle for ECStore { results[results_index_clone] = result; } }); - wait_futs.push(receiver_fut); + wait_futs.push(AbortOnDropHandle::new(receiver_fut)); let scan_plan = ScannerBucketScanPlan { buckets: set_buckets, @@ -318,7 +318,7 @@ impl ScannerIOCycle for ECStore { record_set_scan_failure(&mut first_err, e); } }); - wait_futs.push(scanner_fut); + wait_futs.push(AbortOnDropHandle::new(scanner_fut)); } } diff --git a/crates/utils/src/envs.rs b/crates/utils/src/envs.rs index 2d039dec5..5e572e5c0 100644 --- a/crates/utils/src/envs.rs +++ b/crates/utils/src/envs.rs @@ -268,7 +268,7 @@ where .parse::() .map_err(|_| { log_once(&format!("env_invalid_value:{used_key}"), || { - format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::()) + format!("Invalid {} value for {used_key}. Treating as unset.", type_name::()) }); }) .ok() @@ -570,7 +570,7 @@ where Ok(parsed) => EnvParseOutcome::Parsed(parsed), Err(_) => { log_once(&format!("env_invalid_value:{used_key}"), || { - format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::()) + format!("Invalid {} value for {used_key}. Treating as unset.", type_name::()) }); EnvParseOutcome::Invalid } diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index 9904b41c9..d7ecec7f5 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -12,6 +12,7 @@ for later deletion. ## Open Items +- `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID". - `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations. - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2. - `table-catalog-migration-fence-v1` durable strong migration fence compatibility: version 1 "PREPARING" fences did not distinguish a known-absent global strong snapshot from an unknown baseline, so retries read them but fail closed if the global snapshot is missing. Version 2 preserves the same JSON shape and records the pre-migration global snapshot ETag in the existing target_snapshot_etag field while the fence is "PREPARING". Remove version 1 reads after every supported direct-upgrade source writes version 2 fences and operators have completed or cancelled every older in-progress backing migration. diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index 33206d699..de54b8f40 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -52,7 +52,7 @@ The `/v3/scanner/status` response reports each effective runtime value with a | `scanner.max_wait` | `RUSTFS_SCANNER_MAX_WAIT_SECS` | seconds | preset-derived | Caps one scanner sleep. | | `scanner.cycle` | `RUSTFS_SCANNER_CYCLE` | seconds | preset-derived | Sets the interval between scanner cycles. | | `scanner.start_delay` | `RUSTFS_SCANNER_START_DELAY_SECS` | seconds | unset | Sets startup delay and, for compatibility, the cycle interval when `scanner.cycle` is unset. | -| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `0` | Caps one cycle's runtime. `0` disables this budget. | +| `scanner.cycle_max_duration` | `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` | seconds | `1800` | Caps one cycle's runtime. An explicit `0` disables this budget. | | `scanner.cycle_max_objects` | `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` | objects | `0` | Caps objects processed by one cycle. `0` disables this budget. | | `scanner.cycle_max_directories` | `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` | directories | `0` | Caps directories entered by one cycle. `0` disables this budget. | | `heal.bitrot_cycle` | `RUSTFS_SCANNER_BITROT_CYCLE_SECS` | seconds | `2592000` | Controls periodic deep bitrot scans. `false`, `off`, `no`, or `disabled` disables periodic deep scans; `0`, `true`, `on`, or `yes` runs deep mode every scanner cycle. | @@ -70,6 +70,21 @@ sleep multiplier, maximum wait, and cycle interval. Use `scanner.delay`, `scanner.max_wait`, and `scanner.cycle` when the preset is close but one axis needs a precise override. +When the cycle duration control is unset, RustFS uses a finite 1800-second +(30-minute) default, matching the scanner benchmark guidance. An explicit `0` +preserves the compatibility behavior of an unbounded cycle; object and +directory budgets likewise remain unbounded when explicitly set to `0`. Invalid +or overflowing duration environment values are configuration errors rather than +silent fallback values. + +When a finite deadline expires, RustFS cancels cooperative scanner work and +waits only for the existing bounded shutdown window. A non-yielding I/O future +is dropped after that window. RustFS then attempts a higher leadership epoch so +late cycle, usage, cache, and remote writes from the old generation fail closed. +If the worker cannot stop cooperatively, the cycle state was not confirmed +durable, or that epoch fence cannot be durably persisted, the scanner reports +`recovery-required`; it does not claim an uncooperative cursor was saved. + An explicit `scanner.cycle` or `RUSTFS_SCANNER_CYCLE` is a minimum inter-cycle cadence: dirty-usage notifications do not bypass that configured interval. The default adaptive policy continues to use dirty-usage notifications to wake @@ -144,6 +159,10 @@ metrics.maintenance_control.primary_control metrics.source_work metrics.replication_repair metrics.scan_checkpoint +metrics.cycle_timeout_total +metrics.cycle_last_progress_age +metrics.leader_lease_without_progress +metrics.cycle_recovery_required_total ``` ## Reading Pacing Pressure diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index 7d80a802d..df2fc0847 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -58,7 +58,7 @@ | heal_erasure_disk_rebuild_test | 4 | 🌙 | | inline_fast_path_cluster_test | 16 | | | internode_rpc_signature_e2e_test | 5 | | -| kms | 48 | | +| kms | 46 | | | leading_slash_key_test | 2 | ✅ | | lifecycle_regression_test | 4 | | | list_buckets_auth_test | 1 | ✅ | @@ -99,4 +99,4 @@ | tls_hot_reload_test | 1 | ✅ | | version_id_regression_test | 10 | ✅ | -**Total listed: 577 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 455 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-21. +**Total listed: 575 tests across 82 modules · PR smoke: 163 tests / 36 modules · merge/main full: 453 tests / 73 modules · nightly replication: 55 tests · nightly cluster faults: 28 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-23. diff --git a/protocol/agent/v1/fixtures/fixture-sets.json b/protocol/agent/v1/fixtures/fixture-sets.json index d3beb8f23..72c4c1919 100644 --- a/protocol/agent/v1/fixtures/fixture-sets.json +++ b/protocol/agent/v1/fixtures/fixture-sets.json @@ -25,7 +25,7 @@ }, { "name": "heartbeat", - "status": "reserved", + "status": "populated", "purpose": "Heartbeat payloads, Connect receive time, and freshness window behavior." }, { diff --git a/protocol/agent/v1/fixtures/heartbeat/MANIFEST.sha256 b/protocol/agent/v1/fixtures/heartbeat/MANIFEST.sha256 new file mode 100644 index 000000000..7aa8e3191 --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/MANIFEST.sha256 @@ -0,0 +1,5 @@ +975c1ca53eefeef6766a6fc0b3d3281f7408255342b0686e5e2aee5ad055414c duplicate.json +963529a38a02849c6c2acc6d72668dca9f63218b49c89fae41a451b584850411 overflow.json +e3adeee1c8a19aa17e70894896fb79c072e3785bea3611b93c11e79f039ed5af stale.json +35b9cebd8525389a701e8fe69fbe96407bcb31aa28392fe95babf4a4886985ad unknown.json +37941735dbd6ad3d238258a7b2cae6f0b3aa0ecaae1d8817817c3d718d11d633 valid.json diff --git a/protocol/agent/v1/fixtures/heartbeat/duplicate.json b/protocol/agent/v1/fixtures/heartbeat/duplicate.json new file mode 100644 index 000000000..11cad8ac1 --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/duplicate.json @@ -0,0 +1,9 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "duplicate", + "description": "An exact requestId replay returns the first result and creates no second heartbeat.", + "first": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42}, + "replay": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42}, + "expected": {"decision": "DUPLICATE", "heartbeatWrites": 1, "events": 1, "sameResponse": true} +} diff --git a/protocol/agent/v1/fixtures/heartbeat/overflow.json b/protocol/agent/v1/fixtures/heartbeat/overflow.json new file mode 100644 index 000000000..7b62b5b45 --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/overflow.json @@ -0,0 +1,11 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "overflow", + "description": "Values beyond frozen bounds are rejected before persistence.", + "vectors": [ + {"field": "sequence", "value": 9007199254740992, "maximum": 9007199254740991}, + {"field": "coarseNodeSummary.total", "value": 4097, "maximum": 4096} + ], + "expected": {"decision": "REJECT", "httpStatus": 422, "status": "INVALID_ARGUMENT"} +} diff --git a/protocol/agent/v1/fixtures/heartbeat/stale.json b/protocol/agent/v1/fixtures/heartbeat/stale.json new file mode 100644 index 000000000..aabe98a6f --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/stale.json @@ -0,0 +1,9 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "stale", + "description": "A lower heartbeat sequence is retained as history and cannot replace the current projection.", + "head": {"requestId": "550e8400-e29b-41d4-a716-446655440000", "sequence": 42}, + "late": {"requestId": "7c4d2e10-9f83-4a5b-b6c7-d8e9f0a1b2c3", "sequence": 9}, + "expected": {"decision": "ACCEPT_HISTORY", "currentSequence": 42, "historySequence": 9} +} diff --git a/protocol/agent/v1/fixtures/heartbeat/unknown.json b/protocol/agent/v1/fixtures/heartbeat/unknown.json new file mode 100644 index 000000000..6639cc0ff --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/unknown.json @@ -0,0 +1,18 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "unknown", + "description": "Unknown optional members and capabilities are accepted, discarded before hashing, and never stored or echoed.", + "requestAdditions": { + "telemetryProfile": "extended", + "authorization": "Bearer non-functional-example", + "capabilities": ["heartbeat", "future.capability"], + "coarseNodeSummary": {"rackNames": ["customer-rack"]} + }, + "expected": { + "decision": "ACCEPT", + "storedCapabilities": ["heartbeat"], + "discarded": ["authorization", "future.capability", "telemetryProfile", "coarseNodeSummary.rackNames"], + "echoed": [] + } +} diff --git a/protocol/agent/v1/fixtures/heartbeat/valid.json b/protocol/agent/v1/fixtures/heartbeat/valid.json new file mode 100644 index 000000000..ed3561b80 --- /dev/null +++ b/protocol/agent/v1/fixtures/heartbeat/valid.json @@ -0,0 +1,21 @@ +{ + "protocolVersion": "v1", + "fixtureSet": "heartbeat", + "fixture": "valid", + "description": "A bounded L0 heartbeat. clientTime is advisory; Connect's receivedAt is online authority.", + "request": { + "protocolVersion": "v1", + "requestId": "550e8400-e29b-41d4-a716-446655440000", + "agentVersion": "rustfs-agent/1.19.4", + "capabilities": ["heartbeat", "inventory"], + "sequence": 42, + "clientTime": "2026-08-22T01:02:03Z", + "coarseNodeSummary": {"total": 8, "healthy": 7, "degraded": 1} + }, + "expected": { + "decision": "ACCEPT", + "acceptedVersion": "v1", + "responseFields": ["serverTime", "acceptedVersion", "capabilityHints"], + "onlineAuthority": "serverTime" + } +} diff --git a/rustfs/src/connect/config.rs b/rustfs/src/connect/config.rs new file mode 100644 index 000000000..391d4ef45 --- /dev/null +++ b/rustfs/src/connect/config.rs @@ -0,0 +1,173 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::env; +use std::ffi::OsString; +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +use super::{CredentialStore, IdentityStore}; + +pub const ENV_CONNECT_ENDPOINT: &str = "RUSTFS_CONNECT_ENDPOINT"; +pub const ENV_CONNECT_ROOT_CA_FILE: &str = "RUSTFS_CONNECT_ROOT_CA_FILE"; +pub const ENV_CONNECT_STATE_DIR: &str = "RUSTFS_CONNECT_STATE_DIR"; + +#[derive(Clone, Copy, Debug)] +pub struct HeartbeatSchedule { + pub cadence: Duration, + pub jitter: Duration, + pub timeout: Duration, + pub initial_backoff: Duration, + pub max_backoff: Duration, +} + +impl Default for HeartbeatSchedule { + fn default() -> Self { + Self { + cadence: Duration::from_secs(30), + jitter: Duration::from_secs(3), + timeout: Duration::from_secs(5), + initial_backoff: Duration::from_secs(1), + max_backoff: Duration::from_secs(5 * 60), + } + } +} + +#[derive(Clone, Debug)] +pub struct HeartbeatConfig { + pub endpoint: String, + pub root_ca_pem: Vec, + pub identity_store: IdentityStore, + pub credential_store: CredentialStore, + pub state_path: PathBuf, + pub schedule: HeartbeatSchedule, +} + +impl HeartbeatConfig { + pub fn new( + endpoint: impl Into, + root_ca_pem: impl Into>, + identity_store: IdentityStore, + credential_store: CredentialStore, + state_path: impl Into, + ) -> Self { + Self { + endpoint: endpoint.into(), + root_ca_pem: root_ca_pem.into(), + identity_store, + credential_store, + state_path: state_path.into(), + schedule: HeartbeatSchedule::default(), + } + } + + pub fn from_env() -> Result, HeartbeatConfigError> { + Self::from_env_values( + env::var_os(ENV_CONNECT_ENDPOINT), + env::var_os(ENV_CONNECT_ROOT_CA_FILE), + env::var_os(ENV_CONNECT_STATE_DIR), + ) + } + + fn from_env_values( + endpoint: Option, + root_ca_file: Option, + state_dir: Option, + ) -> Result, HeartbeatConfigError> { + let configured = endpoint.is_some() || root_ca_file.is_some() || state_dir.is_some(); + if !configured { + return Ok(None); + } + let (Some(endpoint), Some(root_ca_file), Some(state_dir)) = (endpoint, root_ca_file, state_dir) else { + return Err(HeartbeatConfigError::Partial); + }; + let endpoint = endpoint.into_string().map_err(|_| HeartbeatConfigError::EndpointEncoding)?; + let root_ca_file = PathBuf::from(root_ca_file); + let state_dir = PathBuf::from(state_dir); + if endpoint.is_empty() || root_ca_file.as_os_str().is_empty() || state_dir.as_os_str().is_empty() { + return Err(HeartbeatConfigError::Partial); + } + let root_ca_pem = fs::read(&root_ca_file).map_err(|source| HeartbeatConfigError::RootCertificate { + path: root_ca_file, + source, + })?; + Ok(Some(Self::new( + endpoint, + root_ca_pem, + IdentityStore::new(state_dir.join("identity")), + CredentialStore::new(state_dir.join("credential")), + state_dir.join("heartbeat/state.json"), + ))) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum HeartbeatConfigError { + #[error( + "Connect heartbeat configuration requires RUSTFS_CONNECT_ENDPOINT, RUSTFS_CONNECT_ROOT_CA_FILE, and RUSTFS_CONNECT_STATE_DIR" + )] + Partial, + #[error("RUSTFS_CONNECT_ENDPOINT is not valid UTF-8")] + EndpointEncoding, + #[error("failed to read the Connect root CA at {path}: {source}")] + RootCertificate { + path: PathBuf, + #[source] + source: std::io::Error, + }, +} + +#[cfg(test)] +mod tests { + use super::{HeartbeatConfig, HeartbeatConfigError}; + use std::ffi::OsString; + + #[test] + fn absent_environment_is_disabled_without_side_effects() { + assert!( + HeartbeatConfig::from_env_values(None, None, None) + .expect("absent config") + .is_none() + ); + } + + #[test] + fn partial_environment_is_rejected() { + assert!(matches!( + HeartbeatConfig::from_env_values(Some(OsString::from("https://connect.example/agent/")), None, None), + Err(HeartbeatConfigError::Partial) + )); + } + + #[test] + fn complete_environment_builds_the_durable_paths() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path().join("root.pem"); + std::fs::write(&root, b"root certificate").expect("root CA"); + let state = temp.path().join("state"); + let config = HeartbeatConfig::from_env_values( + Some(OsString::from("https://connect.example/agent/")), + Some(root.into_os_string()), + Some(state.clone().into_os_string()), + ) + .expect("complete config") + .expect("enabled config"); + + assert_eq!(config.endpoint, "https://connect.example/agent/"); + assert_eq!(config.root_ca_pem, b"root certificate"); + assert_eq!(config.state_path, state.join("heartbeat/state.json")); + assert!(!state.exists(), "parsing configuration must not create state"); + } +} diff --git a/rustfs/src/connect/heartbeat.rs b/rustfs/src/connect/heartbeat.rs new file mode 100644 index 000000000..2eeb8c135 --- /dev/null +++ b/rustfs/src/connect/heartbeat.rs @@ -0,0 +1,585 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fs; +use std::io::{self, Write as _}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use chrono::{DateTime, SecondsFormat, Utc}; +use reqwest::{Client, StatusCode, Url, header}; +use rustls::RootCertStore; +use rustls::pki_types::{CertificateDer, pem::PemObject as _}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; +use zeroize::Zeroizing; + +use super::config::HeartbeatConfig; +use super::credential_store::{CredentialStoreError, DeviceCredential}; +use super::identity::IdentityError; +use super::identity_store::StoreError; +use super::registration::{CredentialValidationError, validate_stored_credential}; + +const PROTOCOL_VERSION: &str = "v1"; +const AGENT_VERSION: &str = concat!("rustfs-agent/", env!("CARGO_PKG_VERSION")); +const MAX_SEQUENCE: u64 = 9_007_199_254_740_991; +const MAX_RESPONSE_BYTES: usize = 64 * 1024; +#[cfg(unix)] +const FILE_MODE: u32 = 0o600; +static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct CoarseNodeSummary { + total: u16, + healthy: u16, + degraded: u16, +} + +impl CoarseNodeSummary { + pub fn new(total: u16, healthy: u16, degraded: u16) -> Result { + let summary = Self { + total, + healthy, + degraded, + }; + if !summary.is_valid() { + return Err(HeartbeatError::NodeSummary); + } + Ok(summary) + } + + fn is_valid(&self) -> bool { + self.total != 0 + && self.total <= 4096 + && self.healthy <= 4096 + && self.degraded <= 4096 + && self.healthy.saturating_add(self.degraded) <= self.total + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum HeartbeatStatus { + Starting, + Online { server_time: String }, + BackingOff { delay: Duration }, + AuthenticationStopped { status: u16, reason: Option }, + Failed { reason: String }, + Stopped, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub(crate) struct PendingHeartbeat { + protocol_version: String, + request_id: String, + agent_version: String, + capabilities: [String; 1], + sequence: u64, + client_time: String, + coarse_node_summary: CoarseNodeSummary, +} + +impl PendingHeartbeat { + fn is_valid(&self) -> bool { + self.protocol_version == PROTOCOL_VERSION + && self.agent_version == AGENT_VERSION + && self.capabilities[0] == "heartbeat" + && self.sequence <= MAX_SEQUENCE + && self.coarse_node_summary.is_valid() + && is_exact_utc_seconds(&self.client_time) + && Uuid::parse_str(&self.request_id) + .is_ok_and(|request_id| request_id.get_version_num() == 4 && request_id.to_string() == self.request_id) + } +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct HeartbeatResponse { + server_time: String, + accepted_version: String, + #[serde(default)] + capability_hints: Vec, +} + +pub(crate) enum Delivery { + Accepted { server_time: String }, + Retry { retry_after: Option }, + AuthenticationStopped { status: u16, reason: Option }, + Rejected { status: u16, reason: Option }, +} + +pub(crate) struct HeartbeatSender { + endpoint: Url, + root_store: RootCertStore, + roots: Vec>, + config: HeartbeatConfig, +} + +impl HeartbeatSender { + pub(crate) fn new(config: HeartbeatConfig) -> Result { + let mut endpoint = Url::parse(&config.endpoint).map_err(|_| HeartbeatError::Endpoint)?; + if endpoint.scheme() != "https" + || endpoint.cannot_be_a_base() + || !endpoint.username().is_empty() + || endpoint.password().is_some() + || endpoint.query().is_some() + || endpoint.fragment().is_some() + { + return Err(HeartbeatError::Endpoint); + } + if !endpoint.path().ends_with('/') { + endpoint.set_path(&format!("{}/", endpoint.path())); + } + let roots = CertificateDer::pem_slice_iter(&config.root_ca_pem) + .collect::, _>>() + .map_err(|_| HeartbeatError::RootCertificate)?; + if roots.is_empty() { + return Err(HeartbeatError::RootCertificate); + } + let mut root_store = RootCertStore::empty(); + let (accepted, rejected) = root_store.add_parsable_certificates(roots.clone()); + if accepted != roots.len() || rejected != 0 { + return Err(HeartbeatError::RootCertificate); + } + let schedule = config.schedule; + if schedule.cadence.is_zero() + || schedule.timeout.is_zero() + || schedule.timeout > Duration::from_secs(5) + || schedule.initial_backoff.is_zero() + || schedule.max_backoff < schedule.initial_backoff + || schedule.max_backoff > Duration::from_secs(5 * 60) + || schedule.jitter > schedule.cadence + { + return Err(HeartbeatError::Schedule); + } + Ok(Self { + endpoint, + root_store, + roots, + config, + }) + } + + pub(crate) async fn send(&self, heartbeat: &PendingHeartbeat) -> Result { + let (cluster_uid, client) = { + let _lock = self.config.credential_store.lock().await?; + let credential = self.config.credential_store.load()?.ok_or(HeartbeatError::NotRegistered)?; + let identity = self.config.identity_store.load()?.ok_or(HeartbeatError::IdentityMissing)?; + validate_stored_credential(&credential, &identity, &self.root_store, &self.roots)?; + let now = Utc::now().timestamp(); + if now < credential.not_before_unix || now >= credential.not_after_unix { + return Err(HeartbeatError::CredentialExpired); + } + let cluster_uid = cluster_uid(&credential)?.to_owned(); + let client = self.client(&credential, &identity.to_pkcs8_pem()?)?; + (cluster_uid, client) + }; + let url = self.endpoint.join(&format!("clusters/{cluster_uid}/heartbeats"))?; + let response = match client.post(url).json(heartbeat).send().await { + Ok(response) => response, + Err(error) if error.is_timeout() || error.is_connect() || error.is_request() => { + return Ok(Delivery::Retry { retry_after: None }); + } + Err(error) => return Err(error.into()), + }; + let status = response.status(); + if status == StatusCode::TOO_MANY_REQUESTS { + return Ok(Delivery::Retry { + retry_after: retry_after(response.headers(), Utc::now(), self.config.schedule.max_backoff), + }); + } + if status == StatusCode::REQUEST_TIMEOUT || status.is_server_error() { + return Ok(Delivery::Retry { retry_after: None }); + } + if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { + return Ok(Delivery::AuthenticationStopped { + status: status.as_u16(), + reason: response_reason(response).await, + }); + } + if status != StatusCode::OK { + return Ok(Delivery::Rejected { + status: status.as_u16(), + reason: response_reason(response).await, + }); + } + let accepted: HeartbeatResponse = + serde_json::from_slice(&bounded_body(response).await?).map_err(|_| HeartbeatError::Response)?; + if accepted.accepted_version != PROTOCOL_VERSION + || accepted.capability_hints.len() > 32 + || accepted.capability_hints.iter().any(|hint| hint.len() > 32) + || !is_exact_utc_seconds(&accepted.server_time) + { + return Err(HeartbeatError::Response); + } + Ok(Delivery::Accepted { + server_time: accepted.server_time, + }) + } + + fn client(&self, credential: &DeviceCredential, key: &Zeroizing) -> Result { + let mut pem = Zeroizing::new(Vec::with_capacity(credential.certificate_chain.len() + key.len() + 1)); + pem.extend_from_slice(credential.certificate_chain.as_bytes()); + pem.push(b'\n'); + pem.extend_from_slice(key.as_bytes()); + let identity = reqwest::Identity::from_pem(&pem).map_err(|_| HeartbeatError::IdentityCertificate)?; + let roots = self + .roots + .iter() + .map(|root| reqwest::Certificate::from_der(root.as_ref())) + .collect::, _>>()?; + Client::builder() + .https_only(true) + .redirect(reqwest::redirect::Policy::none()) + .timeout(self.config.schedule.timeout) + .tls_certs_only(roots) + .identity(identity) + .build() + .map_err(Into::into) + } +} + +#[derive(Clone)] +pub(crate) struct HeartbeatStateStore { + path: PathBuf, +} + +#[derive(Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct HeartbeatState { + next_sequence: u64, + pending: Option, +} + +impl HeartbeatStateStore { + pub(crate) fn new(path: PathBuf) -> Self { + Self { path } + } + + pub(crate) fn try_runtime_lock(&self) -> Result { + let directory = parent(&self.path)?; + fs::create_dir_all(directory).map_err(|source| state_io(directory, source))?; + let name = filename(&self.path)?; + let path = directory.join(format!(".{name}.lock")); + let mut options = fs::OpenOptions::new(); + options.create(true).truncate(false).read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(FILE_MODE); + } + let lock = options.open(&path).map_err(|source| state_io(&path, source))?; + check_mode(&path)?; + lock.try_lock().map_err(|_| HeartbeatError::AlreadyRunning)?; + Ok(lock) + } + + pub(crate) async fn prepare( + &self, + summary: CoarseNodeSummary, + now: DateTime, + ) -> Result { + let store = self.clone(); + tokio::task::spawn_blocking(move || store.prepare_sync(summary, now)) + .await + .map_err(|source| state_io(&self.path, io::Error::other(source)))? + } + + pub(crate) async fn mark_accepted(&self, accepted: &PendingHeartbeat) -> Result<(), HeartbeatError> { + let store = self.clone(); + let accepted = accepted.clone(); + tokio::task::spawn_blocking(move || store.mark_accepted_sync(&accepted)) + .await + .map_err(|source| state_io(&self.path, io::Error::other(source)))? + } + + fn prepare_sync(&self, summary: CoarseNodeSummary, now: DateTime) -> Result { + let mut state = self.read()?; + if let Some(pending) = state.pending { + return Ok(pending); + } + if state.next_sequence > MAX_SEQUENCE { + return Err(HeartbeatError::SequenceExhausted); + } + let pending = PendingHeartbeat { + protocol_version: PROTOCOL_VERSION.to_owned(), + request_id: Uuid::new_v4().to_string(), + agent_version: AGENT_VERSION.to_owned(), + capabilities: ["heartbeat".to_owned()], + sequence: state.next_sequence, + client_time: now.to_rfc3339_opts(SecondsFormat::Secs, true), + coarse_node_summary: summary, + }; + state.pending = Some(pending.clone()); + self.write(&state)?; + Ok(pending) + } + + fn mark_accepted_sync(&self, accepted: &PendingHeartbeat) -> Result<(), HeartbeatError> { + let mut state = self.read()?; + if state.pending.as_ref() != Some(accepted) { + return Err(HeartbeatError::StateConflict); + } + state.next_sequence = accepted.sequence.checked_add(1).ok_or(HeartbeatError::SequenceExhausted)?; + state.pending = None; + self.write(&state) + } + + fn read(&self) -> Result { + let bytes = match fs::read(&self.path) { + Ok(bytes) => bytes, + Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(HeartbeatState::default()), + Err(source) => return Err(state_io(&self.path, source)), + }; + check_mode(&self.path)?; + let state: HeartbeatState = serde_json::from_slice(&bytes).map_err(|source| HeartbeatError::StateInvalid { + path: self.path.clone(), + source, + })?; + if state.next_sequence > MAX_SEQUENCE + 1 + || state + .pending + .as_ref() + .is_some_and(|pending| pending.sequence != state.next_sequence || !pending.is_valid()) + { + return Err(HeartbeatError::StateCorrupt { path: self.path.clone() }); + } + Ok(state) + } + + fn write(&self, state: &HeartbeatState) -> Result<(), HeartbeatError> { + let bytes = serde_json::to_vec(state).map_err(|source| HeartbeatError::StateInvalid { + path: self.path.clone(), + source, + })?; + let directory = parent(&self.path)?; + fs::create_dir_all(directory).map_err(|source| state_io(directory, source))?; + let temp = stage(directory, &self.path, &bytes)?; + let result = fs::rename(&temp, &self.path) + .map_err(|source| state_io(&self.path, source)) + .and_then(|()| fsync_dir(directory).map_err(|source| state_io(directory, source))); + if result.is_err() { + let _ = fs::remove_file(temp); + } + result + } +} + +fn cluster_uid(credential: &DeviceCredential) -> Result<&str, HeartbeatError> { + let mut parts = credential.name.split('/'); + let valid = parts.next() == Some("organizations"); + let organization_uid = parts.next(); + let valid = valid && parts.next() == Some("clusters"); + let cluster_uid = parts.next(); + let valid = valid && parts.next() == Some("clusterDevices"); + let device_uid = parts.next(); + if !valid + || organization_uid.is_none_or(str::is_empty) + || cluster_uid.is_none_or(str::is_empty) + || device_uid != Some(credential.uid.as_str()) + || parts.next().is_some() + { + return Err(HeartbeatError::CredentialName); + } + cluster_uid.ok_or(HeartbeatError::CredentialName) +} + +fn retry_after(headers: &header::HeaderMap, now: DateTime, maximum: Duration) -> Option { + let value = headers.get(header::RETRY_AFTER)?.to_str().ok()?; + let delay = value.parse::().ok().map(Duration::from_secs).or_else(|| { + DateTime::parse_from_rfc2822(value) + .ok() + .and_then(|at| (at.with_timezone(&Utc) - now).to_std().ok()) + })?; + Some(delay.min(maximum)) +} + +fn is_exact_utc_seconds(value: &str) -> bool { + DateTime::parse_from_rfc3339(value).is_ok_and(|time| { + time.offset().local_minus_utc() == 0 + && value.ends_with('Z') + && time.with_timezone(&Utc).to_rfc3339_opts(SecondsFormat::Secs, true) == value + }) +} + +async fn response_reason(response: reqwest::Response) -> Option { + #[derive(Deserialize)] + struct Envelope { + #[serde(default)] + details: Vec, + } + #[derive(Deserialize)] + struct Detail { + #[serde(default)] + reason: String, + } + + serde_json::from_slice::(&bounded_body(response).await.ok()?) + .ok()? + .details + .into_iter() + .find_map(|detail| (!detail.reason.is_empty()).then_some(detail.reason)) +} + +async fn bounded_body(mut response: reqwest::Response) -> Result, HeartbeatError> { + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await? { + if body.len().saturating_add(chunk.len()) > MAX_RESPONSE_BYTES { + return Err(HeartbeatError::ResponseTooLarge); + } + body.extend_from_slice(&chunk); + } + Ok(body) +} + +fn parent(path: &Path) -> Result<&Path, HeartbeatError> { + path.parent() + .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state path has no parent"))) +} + +fn filename(path: &Path) -> Result<&str, HeartbeatError> { + path.file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state filename is invalid"))) +} + +fn stage(directory: &Path, destination: &Path, bytes: &[u8]) -> Result { + let name = filename(destination)?; + loop { + let path = directory.join(format!( + ".{name}.{}.{}.tmp", + std::process::id(), + STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed) + )); + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(FILE_MODE); + } + let mut file = match options.open(&path) { + Ok(file) => file, + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => continue, + Err(source) => return Err(state_io(&path, source)), + }; + if let Err(source) = file.write_all(bytes).and_then(|()| file.sync_all()) { + let _ = fs::remove_file(&path); + return Err(state_io(&path, source)); + } + return Ok(path); + } +} + +fn state_io(path: &Path, source: io::Error) -> HeartbeatError { + HeartbeatError::StateIo { + path: path.to_path_buf(), + source, + } +} + +#[cfg(unix)] +fn check_mode(path: &Path) -> Result<(), HeartbeatError> { + use std::os::unix::fs::PermissionsExt as _; + + let mode = fs::metadata(path) + .map_err(|source| state_io(path, source))? + .permissions() + .mode() + & 0o7777; + if mode != FILE_MODE { + return Err(HeartbeatError::StatePermissions { + path: path.to_path_buf(), + mode, + expected: FILE_MODE, + }); + } + Ok(()) +} + +#[cfg(not(unix))] +fn check_mode(_path: &Path) -> Result<(), HeartbeatError> { + Ok(()) +} + +fn fsync_dir(directory: &Path) -> io::Result<()> { + #[cfg(unix)] + fs::File::open(directory)?.sync_all()?; + #[cfg(not(unix))] + let _ = directory; + Ok(()) +} + +#[derive(Debug, thiserror::Error)] +pub enum HeartbeatError { + #[error("Connect heartbeat endpoint must be an HTTPS base URL without credentials, query, or fragment")] + Endpoint, + #[error("Connect heartbeat root CA configuration is invalid")] + RootCertificate, + #[error("Connect heartbeat schedule is invalid")] + Schedule, + #[error("RustFS is not registered with Connect")] + NotRegistered, + #[error("the Connect device private key is missing")] + IdentityMissing, + #[error("the stored Connect certificate and device private key cannot form a TLS identity")] + IdentityCertificate, + #[error("the stored Connect credential name is invalid")] + CredentialName, + #[error("the stored Connect device certificate is not currently valid")] + CredentialExpired, + #[error("the Connect heartbeat node summary is outside protocol bounds")] + NodeSummary, + #[error("the Connect heartbeat sequence is exhausted")] + SequenceExhausted, + #[error("a Connect heartbeat runtime already owns this state")] + AlreadyRunning, + #[error("the persisted Connect heartbeat changed while delivery was in flight")] + StateConflict, + #[error("Connect heartbeat state I/O failed at {path}: {source}")] + StateIo { + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("Connect heartbeat state at {path} is invalid: {source}")] + StateInvalid { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + #[error("Connect heartbeat state at {path} violates the protocol invariants")] + StateCorrupt { path: PathBuf }, + #[cfg(unix)] + #[error("Connect heartbeat state at {path} has mode {mode:o}, expected {expected:o}")] + StatePermissions { path: PathBuf, mode: u32, expected: u32 }, + #[error("Connect heartbeat response exceeded 64 KiB")] + ResponseTooLarge, + #[error("Connect returned an invalid heartbeat response")] + Response, + #[error(transparent)] + Url(#[from] url::ParseError), + #[error(transparent)] + Transport(#[from] reqwest::Error), + #[error(transparent)] + Identity(#[from] IdentityError), + #[error(transparent)] + IdentityStore(#[from] StoreError), + #[error(transparent)] + CredentialStore(#[from] CredentialStoreError), + #[error(transparent)] + CredentialValidation(#[from] CredentialValidationError), +} diff --git a/rustfs/src/connect/mod.rs b/rustfs/src/connect/mod.rs index 350cceb38..3ce2ad975 100644 --- a/rustfs/src/connect/mod.rs +++ b/rustfs/src/connect/mod.rs @@ -21,20 +21,26 @@ //! canonical transcript frozen by //! `protocol/agent/v1/registration-proof.md`. //! -//! Nothing here contacts the network or starts a task. A deployment that has -//! not been enrolled into a Connect control plane never calls into it, so an -//! unconfigured server generates no key and holds no identity. +//! Enrolled deployments may start the optional outbound heartbeat runtime. +//! An unconfigured server starts no Connect task, generates no key, and holds +//! no Connect identity. pub mod client; +pub mod config; pub mod credential_store; +pub mod heartbeat; pub mod identity; pub mod identity_store; pub mod offline; pub mod registration; +pub mod runtime; pub use client::{ClientError, ConnectClient, ConnectConfig}; +pub use config::{HeartbeatConfig, HeartbeatConfigError, HeartbeatSchedule}; pub use credential_store::{CredentialStore, DeviceCredential}; +pub use heartbeat::{CoarseNodeSummary, HeartbeatError, HeartbeatStatus}; pub use identity::{DeviceIdentity, IdentityError, RegistrationProof, RegistrationTranscript}; pub use identity_store::{IdentityStore, StoreError}; pub use offline::{EnrollmentError, OfflineEnrollment, OfflineKeyStore, VerifiedChallenge}; pub use registration::{RegistrationToken, TokenError}; +pub use runtime::{HeartbeatRuntime, spawn_heartbeat_runtime}; diff --git a/rustfs/src/connect/runtime.rs b/rustfs/src/connect/runtime.rs new file mode 100644 index 000000000..f0d919a07 --- /dev/null +++ b/rustfs/src/connect/runtime.rs @@ -0,0 +1,156 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::future::Future; +use std::time::Duration; + +use chrono::Utc; +use rand::RngExt as _; +use tokio::sync::watch; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use super::config::HeartbeatConfig; +use super::heartbeat::{CoarseNodeSummary, Delivery, HeartbeatError, HeartbeatSender, HeartbeatStateStore, HeartbeatStatus}; + +pub struct HeartbeatRuntime { + shutdown: CancellationToken, + status: watch::Receiver, + task: Option>, +} + +impl HeartbeatRuntime { + pub fn status(&self) -> watch::Receiver { + self.status.clone() + } + + pub async fn shutdown(mut self) { + self.shutdown.cancel(); + if let Some(task) = self.task.take() { + let _ = task.await; + } + } +} + +impl Drop for HeartbeatRuntime { + fn drop(&mut self) { + self.shutdown.cancel(); + } +} + +pub fn spawn_heartbeat_runtime( + config: Option, + parent_shutdown: &CancellationToken, + sample: F, +) -> Result, HeartbeatError> +where + F: Fn() -> CoarseNodeSummary + Send + Sync + 'static, +{ + let Some(config) = config else { + return Ok(None); + }; + let sender = HeartbeatSender::new(config.clone())?; + let store = HeartbeatStateStore::new(config.state_path.clone()); + let lock = store.try_runtime_lock()?; + let schedule = config.schedule; + let shutdown = parent_shutdown.child_token(); + let task_shutdown = shutdown.clone(); + let (status_tx, status_rx) = watch::channel(HeartbeatStatus::Starting); + let task = tokio::spawn(async move { + let _lock = lock; + let mut backoff = schedule.initial_backoff; + loop { + if task_shutdown.is_cancelled() { + break; + } + let pending = match store.prepare(sample(), Utc::now()).await { + Ok(pending) => pending, + Err(error) => return failed(&status_tx, error), + }; + let delivery = match cancellable(&task_shutdown, sender.send(&pending)).await { + Some(Ok(delivery)) => delivery, + Some(Err(error)) => return failed(&status_tx, error), + None => break, + }; + let delay = match delivery { + Delivery::Accepted { server_time } => { + if let Err(error) = store.mark_accepted(&pending).await { + return failed(&status_tx, error); + } + backoff = schedule.initial_backoff; + let _ = status_tx.send(HeartbeatStatus::Online { server_time }); + schedule.cadence.saturating_add(jitter(schedule.jitter)) + } + Delivery::Retry { retry_after } => { + let delay = retry_after + .unwrap_or(backoff) + .clamp(schedule.initial_backoff, schedule.max_backoff); + backoff = backoff.saturating_mul(2).min(schedule.max_backoff); + let _ = status_tx.send(HeartbeatStatus::BackingOff { delay }); + delay + } + Delivery::AuthenticationStopped { status, reason } => { + let _ = status_tx.send(HeartbeatStatus::AuthenticationStopped { status, reason }); + return; + } + Delivery::Rejected { status, reason } => { + let suffix = reason.map_or_else(String::new, |reason| format!("; reason={reason}")); + let _ = status_tx.send(HeartbeatStatus::Failed { + reason: format!("Connect rejected heartbeat with HTTP {status}{suffix}"), + }); + return; + } + }; + if sleep_or_cancel(&task_shutdown, delay).await { + break; + } + } + let _ = status_tx.send(HeartbeatStatus::Stopped); + }); + Ok(Some(HeartbeatRuntime { + shutdown, + status: status_rx, + task: Some(task), + })) +} + +fn failed(status: &watch::Sender, error: HeartbeatError) { + let _ = status.send(HeartbeatStatus::Failed { + reason: error.to_string(), + }); +} + +fn jitter(maximum: Duration) -> Duration { + if maximum.is_zero() { + Duration::ZERO + } else { + maximum.mul_f64(rand::rng().random_range(0.0..=1.0)) + } +} + +async fn cancellable(shutdown: &CancellationToken, future: impl Future) -> Option { + tokio::select! { + biased; + () = shutdown.cancelled() => None, + value = future => Some(value), + } +} + +async fn sleep_or_cancel(shutdown: &CancellationToken, delay: Duration) -> bool { + tokio::select! { + biased; + () = shutdown.cancelled() => true, + () = tokio::time::sleep(delay) => false, + } +} diff --git a/rustfs/src/server/readiness.rs b/rustfs/src/server/readiness.rs index d3954b7a7..8a934e06e 100644 --- a/rustfs/src/server/readiness.rs +++ b/rustfs/src/server/readiness.rs @@ -20,6 +20,7 @@ use crate::storage_api::server::readiness::contract::admin::StorageAdminApi; use crate::storage_api::server::readiness::{Endpoint, EndpointServerPools, is_dist_erasure}; #[cfg(test)] use crate::storage_api::server::readiness::{Endpoints, PoolEndpoints}; +use crate::storage_api::startup::shutdown::mark_get_metadata_read_version_coalescing_service_ready; use bytes::Bytes; use http::HeaderValue; use http::{Request as HttpRequest, Response, StatusCode}; @@ -212,6 +213,9 @@ where if readiness_gate_blocks_path(path, &readiness) { return Ok(service_not_ready_response(readiness.current_stage())); } + if !is_probe_path(path) && readiness.is_ready() { + mark_get_metadata_read_version_coalescing_service_ready(); + } let resp = inner.call(req).await?; // System is ready, forward to the actual S3/RPC handlers // Transparently converts any response body into a BoxBody, and then Trace/Cors/Compression continues to work @@ -232,6 +236,7 @@ pub async fn publish_ready_when_runtime_ready( collect_node_readiness, |dependency_readiness| { readiness.mark_stage(rustfs_common::SystemStage::FullReady); + mark_get_metadata_read_version_coalescing_service_ready(); if let Some(state_manager) = state_manager { state_manager.update(ServiceState::Ready); } diff --git a/rustfs/src/startup_lifecycle.rs b/rustfs/src/startup_lifecycle.rs index c4d09872b..ad5681366 100644 --- a/rustfs/src/startup_lifecycle.rs +++ b/rustfs/src/startup_lifecycle.rs @@ -128,6 +128,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec } = lifecycle; let StartupServiceRuntime { optional_runtimes, + heartbeat, iam_bootstrap, enable_scanner, } = service_runtime; @@ -162,6 +163,9 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec shutdown_token, ) .await; + if let Some(heartbeat) = heartbeat { + heartbeat.shutdown().await; + } if let Err(err) = event_notifier_reconciler.await { tracing::warn!( target: "rustfs::main::run", diff --git a/rustfs/src/startup_services.rs b/rustfs/src/startup_services.rs index 585e9677a..c07db0f67 100644 --- a/rustfs/src/startup_services.rs +++ b/rustfs/src/startup_services.rs @@ -16,6 +16,7 @@ use crate::site_replication_reconcile::spawn_site_replication_reconcile_task; use crate::storage_api::startup::services::{ECStore, EndpointServerPools, ServerContextSlot}; use crate::{ config::Config, + connect::{CoarseNodeSummary, HeartbeatConfig, HeartbeatRuntime, spawn_heartbeat_runtime}, init::{init_buffer_profile_system, init_kms_system}, server::ServiceStateManager, startup_audit::init_audit_runtime, @@ -35,6 +36,7 @@ use tokio_util::sync::CancellationToken; pub(crate) struct StartupServiceRuntime { pub(crate) optional_runtimes: OptionalRuntimeServices, + pub(crate) heartbeat: Option, pub(crate) iam_bootstrap: IamBootstrapDisposition, pub(crate) enable_scanner: bool, } @@ -73,6 +75,8 @@ pub(crate) async fn init_startup_runtime_services( init_kms_system(config).await?; let optional_runtimes = init_optional_runtime_services().await?; + let heartbeat_config = HeartbeatConfig::from_env().map_err(std::io::Error::other)?; + let heartbeat_nodes = heartbeat_config.as_ref().map(|_| endpoint_pools.get_nodes().len()); init_buffer_profile_system(config); init_deadlock_detector_runtime(); @@ -92,10 +96,27 @@ pub(crate) async fn init_startup_runtime_services( init_notification_runtime(endpoint_pools, buckets).await?; let enable_scanner = init_background_service_runtime(store.clone()).await?; init_observability_runtime(store.clone(), ctx.clone()).await; + let heartbeat = start_heartbeat_runtime(heartbeat_config, heartbeat_nodes, &ctx)?; Ok(StartupServiceRuntime { optional_runtimes, + heartbeat, iam_bootstrap, enable_scanner, }) } + +fn start_heartbeat_runtime( + config: Option, + node_count: Option, + shutdown: &CancellationToken, +) -> Result> { + let Some(config) = config else { + return Ok(None); + }; + let summary = u16::try_from(node_count.unwrap_or_default()) + .ok() + .and_then(|total| CoarseNodeSummary::new(total, 0, 0).ok()) + .ok_or_else(|| std::io::Error::other("Connect heartbeat node count is outside protocol bounds"))?; + spawn_heartbeat_runtime(Some(config), shutdown, move || summary).map_err(std::io::Error::other) +} diff --git a/rustfs/src/storage/rpc/node_service/disk.rs b/rustfs/src/storage/rpc/node_service/disk.rs index de80a10f0..617d0b80d 100644 --- a/rustfs/src/storage/rpc/node_service/disk.rs +++ b/rustfs/src/storage/rpc/node_service/disk.rs @@ -23,10 +23,12 @@ use bytes::Bytes; use rustfs_filemeta::FileInfo; use rustfs_io_metrics::internode_metrics::{ INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_REQUEST, - INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_OPERATION_GRPC_WRITE_ALL, - INTERNODE_STAGE_READ_VERSION_DISK_READ, INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, - INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, - INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics, + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_VERSION, + INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ, + INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE, + INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE, INTERNODE_STAGE_READ_VERSION_DISK_READ, + INTERNODE_STAGE_READ_VERSION_REQUEST_DECODE, INTERNODE_STAGE_READ_VERSION_RESPONSE_JSON_ENCODE, + INTERNODE_STAGE_READ_VERSION_RESPONSE_MSGPACK_ENCODE, INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics, }; use rustfs_protos::proto_gen::node_service::*; use serde::de::DeserializeOwned; @@ -242,24 +244,42 @@ fn record_read_version_stage(stage: &'static str, started_at: Option) { } } +fn record_batch_read_version_stage(stage: &'static str, started_at: Option) { + if let Some(started_at) = started_at { + global_internode_metrics().record_stage_duration_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + stage, + started_at.elapsed(), + ); + } +} + fn encode_batch_read_version_response_payloads( batch_read_version_resps: &[BatchReadVersionResp], request_decoded_from_msgpack: bool, ) -> std::result::Result<(Vec, Vec), DiskError> { + let attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); let mut batch_read_version_resps_json = Vec::with_capacity(batch_read_version_resps.len()); - let mut batch_read_version_resps_bin = Vec::with_capacity(batch_read_version_resps.len()); - + let json_encode_started = internode_stage_timer(attribution_enabled); for batch_read_version_resp in batch_read_version_resps { batch_read_version_resps_json.push( compat_response_json(batch_read_version_resp, request_decoded_from_msgpack) .map_err(|err| DiskError::other(format!("encode BatchReadVersionResp json failed: {err}")))?, ); + } + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_JSON_ENCODE, json_encode_started); + + let mut batch_read_version_resps_bin = Vec::with_capacity(batch_read_version_resps.len()); + let msgpack_encode_started = internode_stage_timer(attribution_enabled); + for batch_read_version_resp in batch_read_version_resps { batch_read_version_resps_bin.push(Bytes::from(encode_msgpack_with_capacity( batch_read_version_resp, "BatchReadVersionResp", FILE_INFO_MSGPACK_ENCODE_CAPACITY_HINT, )?)); } + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MSGPACK_ENCODE, msgpack_encode_started); Ok((batch_read_version_resps_json, batch_read_version_resps_bin)) } @@ -485,15 +505,37 @@ impl NodeService { &self, request: Request, ) -> Result, Status> { + let attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); let request = request.into_inner(); + if attribution_enabled { + let metrics = global_internode_metrics(); + metrics.record_incoming_request_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + ); + metrics.record_recv_bytes_for_operation_and_backend( + INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, + INTERNODE_TRANSPORT_BACKEND_GRPC, + request + .disk + .len() + .saturating_add(request.batch_read_version_req.len()) + .saturating_add(request.batch_read_version_req_bin.len()), + ); + } if let Some(disk) = self.find_disk(&request.disk).await { + let decode_started = internode_stage_timer(attribution_enabled); let decoded_batch_read_version_req: DecodedRpcPayload = match decode_msgpack_or_json_with_source( &request.batch_read_version_req_bin, &request.batch_read_version_req, "BatchReadVersionReq", ) { - Ok(batch_read_version_req) => batch_read_version_req, + Ok(batch_read_version_req) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE, decode_started); + batch_read_version_req + } Err(err) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_DECODE, decode_started); return Ok(Response::new(BatchReadVersionResponse { success: false, batch_read_version_resps: Vec::new(), @@ -514,8 +556,10 @@ impl NodeService { })); } + let disk_read_started = internode_stage_timer(attribution_enabled); match disk.batch_read_version(batch_read_version_req).await { Ok(batch_read_version_resps) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ, disk_read_started); let (batch_read_version_resps, batch_read_version_resps_bin) = match encode_batch_read_version_response_payloads(&batch_read_version_resps, request_decoded_from_msgpack) { @@ -537,12 +581,15 @@ impl NodeService { error: None, })) } - Err(err) => Ok(Response::new(BatchReadVersionResponse { - success: false, - batch_read_version_resps: Vec::new(), - batch_read_version_resps_bin: Vec::new(), - error: Some(err.into()), - })), + Err(err) => { + record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_DISK_READ, disk_read_started); + Ok(Response::new(BatchReadVersionResponse { + success: false, + batch_read_version_resps: Vec::new(), + batch_read_version_resps_bin: Vec::new(), + error: Some(err.into()), + })) + } } } else { Ok(Response::new(BatchReadVersionResponse { @@ -722,9 +769,9 @@ impl NodeService { &self, request: Request, ) -> Result, Status> { - let request = request.into_inner(); let metrics = global_internode_metrics(); let read_version_attribution_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); + let request = request.into_inner(); if read_version_attribution_enabled { metrics.record_incoming_request_for_operation_and_backend( INTERNODE_OPERATION_GRPC_READ_VERSION, @@ -1635,6 +1682,7 @@ mod tests { encode_batch_read_version_response_payloads, encode_delete_versions_errors, encode_file_info_msgpack, encode_msgpack, encode_msgpack_named, encode_read_multiple_response_payloads, encode_rename_data_response_payloads, }; + use crate::storage::DiskError; use crate::storage::rpc::node_service::make_server; use crate::storage::storage_api::ReadMultipleResp; use crate::storage::storage_api::RenameDataResp; @@ -2028,7 +2076,8 @@ mod tests { path: "object-a".to_string(), version_id: "version-a".to_string(), success: false, - error: "file version not found".to_string(), + error: DiskError::FileVersionNotFound.to_string(), + error_code: DiskError::FileVersionNotFound.to_u32(), ..Default::default() }]; @@ -2044,8 +2093,43 @@ mod tests { .expect("msgpack batch read version response should decode"); assert_eq!(json_decoded.index, responses[0].index); + assert_eq!(json_decoded.error_code, responses[0].error_code); assert_eq!(msgpack_decoded.path, responses[0].path); assert_eq!(msgpack_decoded.error, responses[0].error); + assert_eq!(msgpack_decoded.error_code, responses[0].error_code); + } + + #[test] + fn batch_read_version_response_decode_accepts_legacy_payload_without_error_code() { + #[derive(Serialize)] + struct LegacyBatchReadVersionResp { + index: usize, + path: String, + version_id: String, + success: bool, + file_info: FileInfo, + error: String, + } + + let legacy = LegacyBatchReadVersionResp { + index: 2, + path: "object-legacy".to_string(), + version_id: "version-legacy".to_string(), + success: false, + file_info: FileInfo::default(), + error: "legacy error".to_string(), + }; + let legacy_json = serde_json::to_string(&legacy).expect("legacy json should encode"); + let legacy_msgpack = encode_msgpack(&legacy, "LegacyBatchReadVersionResp").expect("legacy msgpack should encode"); + + let json_decoded: BatchReadVersionResp = + decode_msgpack_or_json(&[], &legacy_json, "BatchReadVersionResp").expect("legacy json should decode"); + let msgpack_decoded: BatchReadVersionResp = + decode_msgpack_or_json(&legacy_msgpack, "", "BatchReadVersionResp").expect("legacy msgpack should decode"); + + assert_eq!(json_decoded.error_code, 0); + assert_eq!(msgpack_decoded.error_code, 0); + assert_eq!(msgpack_decoded.error, legacy.error); } #[test] diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index e5277c0dd..512e255ee 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -1100,6 +1100,10 @@ pub(crate) fn shutdown_background_monitors() { rustfs_ecstore::shutdown_background_monitors(); } +pub(crate) fn mark_get_metadata_read_version_coalescing_service_ready() { + rustfs_ecstore::mark_get_metadata_read_version_coalescing_service_ready(); +} + pub(crate) fn set_global_rustfs_port(value: u16) { ecstore_global::set_global_rustfs_port(value); } diff --git a/rustfs/src/storage_api.rs b/rustfs/src/storage_api.rs index e90c1f528..dea2923ac 100644 --- a/rustfs/src/storage_api.rs +++ b/rustfs/src/storage_api.rs @@ -284,7 +284,8 @@ pub(crate) mod startup { pub(crate) mod shutdown { pub(crate) use crate::storage::storage_api::{ - shutdown_background_monitors, shutdown_background_services, store_compression_total_in_backend, + mark_get_metadata_read_version_coalescing_service_ready, shutdown_background_monitors, shutdown_background_services, + store_compression_total_in_backend, }; } diff --git a/rustfs/tests/connect_heartbeat.rs b/rustfs/tests/connect_heartbeat.rs new file mode 100644 index 000000000..9414cce46 --- /dev/null +++ b/rustfs/tests/connect_heartbeat.rs @@ -0,0 +1,687 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::VecDeque; +use std::fs; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::Bytes; +use http_body_util::{BodyExt as _, Full}; +use hyper::service::service_fn; +use hyper::{Request, Response, StatusCode}; +use hyper_util::rt::TokioIo; +use rcgen::{ + BasicConstraints, CertificateParams, DistinguishedName, DnType, ExtendedKeyUsagePurpose, IsCa, Issuer, KeyPair, + KeyUsagePurpose, SanType, +}; +use rustfs::connect::{ + CoarseNodeSummary, CredentialStore, DeviceCredential, HeartbeatConfig, HeartbeatSchedule, HeartbeatStatus, IdentityStore, + spawn_heartbeat_runtime, +}; +use rustls::RootCertStore; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; +use rustls::server::WebPkiClientVerifier; +use serde_json::{Value, json}; +use time::OffsetDateTime; +use tokio::net::TcpListener; +use tokio::sync::watch; +use tokio_rustls::TlsAcceptor; +use tokio_util::sync::CancellationToken; + +const ORGANIZATION_UID: &str = "0198f4b0-1a00-7c10-8d21-2e3f4a5b6c70"; +const CLUSTER_UID: &str = "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81"; +const DEVICE_UID: &str = "0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92"; + +struct TestPki { + root_params: CertificateParams, + root_key: KeyPair, + root_der: CertificateDer<'static>, + root_pem: String, + server_der: CertificateDer<'static>, + server_key: PrivatePkcs8KeyDer<'static>, +} + +impl TestPki { + fn new() -> Self { + let now = OffsetDateTime::now_utc(); + let root_key = KeyPair::generate().expect("generate root key"); + let mut root_params = CertificateParams::default(); + root_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + root_params.not_before = now - time::Duration::days(30); + root_params.not_after = now + time::Duration::days(30); + root_params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::DigitalSignature]; + let root = root_params.self_signed(&root_key).expect("sign root"); + + let server_key = KeyPair::generate().expect("generate server key"); + let mut server_params = CertificateParams::default(); + server_params.not_before = now - time::Duration::hours(1); + server_params.not_after = now + time::Duration::days(2); + server_params + .subject_alt_names + .push(SanType::DnsName("localhost".try_into().expect("valid DNS name"))); + server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let server = server_params + .signed_by(&server_key, &Issuer::from_params(&root_params, &root_key)) + .expect("sign server certificate"); + Self { + root_params, + root_key, + root_der: root.der().clone(), + root_pem: root.pem(), + server_der: server.der().clone(), + server_key: PrivatePkcs8KeyDer::from(server_key.serialize_der()), + } + } + + fn server_config(&self) -> rustls::ServerConfig { + let mut roots = RootCertStore::empty(); + roots.add(self.root_der.clone()).expect("add client root"); + let verifier = WebPkiClientVerifier::builder(Arc::new(roots)) + .build() + .expect("client verifier"); + rustls::ServerConfig::builder() + .with_client_cert_verifier(verifier) + .with_single_cert(vec![self.server_der.clone()], PrivateKeyDer::Pkcs8(self.server_key.clone_key())) + .expect("server TLS") + } + + fn stores(&self, temp: &tempfile::TempDir) -> (IdentityStore, CredentialStore) { + let now = OffsetDateTime::now_utc(); + self.stores_with_certificate(temp, now - time::Duration::hours(1), now + time::Duration::hours(23), true) + } + + fn stores_with_certificate( + &self, + temp: &tempfile::TempDir, + not_before: OffsetDateTime, + not_after: OffsetDateTime, + bind_identity: bool, + ) -> (IdentityStore, CredentialStore) { + let identity_store = IdentityStore::new(temp.path().join("identity")); + let identity = identity_store.load_or_create().expect("create identity"); + let private_key = PrivatePkcs8KeyDer::from(identity.to_pkcs8_der().expect("serialize key").to_vec()); + let device_key = if bind_identity { + KeyPair::from_pkcs8_der_and_sign_algo(&private_key, &rcgen::PKCS_ECDSA_P256_SHA256).expect("device key") + } else { + KeyPair::generate().expect("mismatched device key") + }; + let mut params = CertificateParams::default(); + params.not_before = not_before; + params.not_after = not_after; + params.serial_number = Some(vec![1; 16].into()); + params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ClientAuth]; + params.distinguished_name = DistinguishedName::new(); + params.distinguished_name.push(DnType::CommonName, DEVICE_UID); + params.subject_alt_names.push(SanType::URI( + format!("urn:rustfs:connect:device:{DEVICE_UID}") + .try_into() + .expect("device URI"), + )); + let certificate = params + .signed_by(&device_key, &Issuer::from_params(&self.root_params, &self.root_key)) + .expect("device certificate"); + let cluster = format!("organizations/{ORGANIZATION_UID}/clusters/{CLUSTER_UID}"); + let credential = DeviceCredential { + name: format!("{cluster}/clusterDevices/{DEVICE_UID}"), + uid: DEVICE_UID.to_owned(), + protocol_version: "v1".to_owned(), + key_id: format!("x509-{}", "01".repeat(16)), + certificate_serial: "01".repeat(16), + certificate: certificate.pem(), + certificate_chain: certificate.pem(), + not_before_unix: not_before.unix_timestamp(), + not_after_unix: not_after.unix_timestamp(), + }; + let directory = temp.path().join("credential"); + fs::create_dir_all(&directory).expect("credential directory"); + let path = directory.join("device.crt.json"); + fs::write(&path, serde_json::to_vec(&credential).expect("credential JSON")).expect("write credential"); + private_mode(&path); + (identity_store, CredentialStore::new(directory)) + } +} + +#[derive(Clone)] +struct Reply { + status: StatusCode, + body: Value, + retry_after: Option<&'static str>, + delay: Duration, +} + +impl Reply { + fn ok(time: &str) -> Self { + Self { + status: StatusCode::OK, + body: json!({ + "serverTime": time, + "acceptedVersion": "v1", + "capabilityHints": [], + "futureField": true + }), + retry_after: None, + delay: Duration::ZERO, + } + } + + fn error(status: StatusCode) -> Self { + Self { + status, + body: json!({"details": []}), + retry_after: None, + delay: Duration::ZERO, + } + } +} + +struct TestServer { + endpoint: String, + seen: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn server(pki: &TestPki, replies: Vec) -> TestServer { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind server"); + let address = listener.local_addr().expect("server address"); + let acceptor = TlsAcceptor::from(Arc::new(pki.server_config())); + let replies = Arc::new(Mutex::new(VecDeque::from(replies))); + let seen = Arc::new(Mutex::new(Vec::new())); + let captured = seen.clone(); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + let acceptor = acceptor.clone(); + let replies = replies.clone(); + let seen = captured.clone(); + tokio::spawn(async move { + let Ok(stream) = acceptor.accept(stream).await else { return }; + let service = service_fn(move |request: Request| { + let replies = replies.clone(); + let seen = seen.clone(); + async move { + assert_eq!(request.uri().path(), format!("/agent/clusters/{CLUSTER_UID}/heartbeats")); + let body = request.into_body().collect().await.expect("request body").to_bytes(); + seen.lock() + .expect("seen lock") + .push(serde_json::from_slice(&body).expect("request JSON")); + let reply = replies + .lock() + .expect("reply lock") + .pop_front() + .unwrap_or_else(|| Reply::error(StatusCode::SERVICE_UNAVAILABLE)); + if !reply.delay.is_zero() { + tokio::time::sleep(reply.delay).await; + } + let mut builder = Response::builder() + .status(reply.status) + .header("content-type", "application/json"); + if let Some(value) = reply.retry_after { + builder = builder.header("retry-after", value); + } + Ok::<_, hyper::Error>( + builder + .body(Full::new(Bytes::from(serde_json::to_vec(&reply.body).expect("reply JSON")))) + .expect("reply"), + ) + } + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }); + } + }); + TestServer { + endpoint: format!("https://localhost:{}/agent/", address.port()), + seen, + task, + } +} + +fn config(temp: &tempfile::TempDir, pki: &TestPki, server: &TestServer) -> HeartbeatConfig { + let (identity_store, credential_store) = pki.stores(temp); + config_with_stores(temp, pki, server, identity_store, credential_store) +} + +fn config_with_stores( + temp: &tempfile::TempDir, + pki: &TestPki, + server: &TestServer, + identity_store: IdentityStore, + credential_store: CredentialStore, +) -> HeartbeatConfig { + HeartbeatConfig { + endpoint: server.endpoint.clone(), + root_ca_pem: pki.root_pem.as_bytes().to_vec(), + identity_store, + credential_store, + state_path: temp.path().join("heartbeat/state.json"), + schedule: HeartbeatSchedule { + cadence: Duration::from_millis(40), + jitter: Duration::ZERO, + timeout: Duration::from_millis(200), + initial_backoff: Duration::from_millis(20), + max_backoff: Duration::from_millis(80), + }, + } +} + +fn rewrite_credential(temp: &tempfile::TempDir, update: impl FnOnce(&mut DeviceCredential)) { + let path = temp.path().join("credential/device.crt.json"); + let mut credential: DeviceCredential = + serde_json::from_slice(&fs::read(&path).expect("read credential")).expect("parse credential"); + update(&mut credential); + fs::write(&path, serde_json::to_vec(&credential).expect("credential JSON")).expect("rewrite credential"); + private_mode(&path); +} + +fn summary() -> CoarseNodeSummary { + CoarseNodeSummary::new(8, 7, 1).expect("node summary") +} + +async fn wait_for( + status: &mut watch::Receiver, + predicate: impl Fn(&HeartbeatStatus) -> bool, +) -> HeartbeatStatus { + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let current = status.borrow_and_update().clone(); + if predicate(¤t) { + return current; + } + status.changed().await.expect("status channel"); + } + }) + .await + .expect("heartbeat status timeout") +} + +async fn assert_credential_failure(config: HeartbeatConfig, server: &TestServer, expected: &str) { + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + assert!(matches!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await, + HeartbeatStatus::Failed { reason } if reason.contains(expected) + )); + assert!(server.seen.lock().expect("seen lock").is_empty()); + runtime.shutdown().await; +} + +#[tokio::test] +async fn connect_config_absent_starts_no_task() { + let shutdown = CancellationToken::new(); + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let sampled = calls.clone(); + let runtime = spawn_heartbeat_runtime(None, &shutdown, move || { + sampled.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + summary() + }) + .expect("absent config"); + + assert!(runtime.is_none()); + tokio::task::yield_now().await; + assert_eq!(calls.load(std::sync::atomic::Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn duplicate_runtime_is_rejected_without_a_second_task() { + let pki = TestPki::new(); + let mut reply = Reply::ok("2026-08-22T01:02:03Z"); + reply.delay = Duration::from_secs(5); + let server = server(&pki, vec![reply]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let config = config(&temp, &pki, &server); + let runtime = spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary) + .expect("first runtime") + .expect("configured runtime"); + + assert!(matches!( + spawn_heartbeat_runtime(Some(config), &shutdown, summary), + Err(rustfs::connect::HeartbeatError::AlreadyRunning) + )); + runtime.shutdown().await; +} + +#[tokio::test(flavor = "current_thread")] +async fn dropped_runtime_keeps_the_lock_until_its_task_stops() { + let pki = TestPki::new(); + let server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z")]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let config = config(&temp, &pki, &server); + let runtime = spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary) + .expect("first runtime") + .expect("configured runtime"); + + drop(runtime); + assert!(matches!( + spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary), + Err(rustfs::connect::HeartbeatError::AlreadyRunning) + )); + + let replacement = tokio::time::timeout(Duration::from_secs(3), async { + loop { + match spawn_heartbeat_runtime(Some(config.clone()), &shutdown, summary) { + Ok(Some(runtime)) => break runtime, + Err(rustfs::connect::HeartbeatError::AlreadyRunning) => tokio::task::yield_now().await, + Ok(None) => panic!("configured replacement returned no runtime"), + Err(error) => panic!("unexpected replacement error: {error}"), + } + } + }) + .await + .expect("dropped runtime releases its lock after stopping"); + replacement.shutdown().await; +} + +#[tokio::test] +async fn corrupt_persisted_state_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z")]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let config = config(&temp, &pki, &server); + let directory = config.state_path.parent().expect("state directory"); + fs::create_dir_all(directory).expect("create state directory"); + fs::write( + &config.state_path, + br#"{"nextSequence":0,"pending":{"protocolVersion":"v1","requestId":"550e8400-e29b-41d4-a716-446655440000","agentVersion":"rustfs-agent/1.0.0-rc.3","capabilities":["heartbeat"],"sequence":0,"clientTime":"2026-08-22T01:02:03Z","coarseNodeSummary":{"total":0,"healthy":0,"degraded":0}}}"#, + ) + .expect("write corrupt state"); + private_mode(&config.state_path); + let runtime = spawn_heartbeat_runtime(Some(config), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + + assert!(matches!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Failed { .. })).await, + HeartbeatStatus::Failed { reason } if reason.contains("violates the protocol invariants") + )); + assert!(server.seen.lock().expect("seen lock").is_empty()); + runtime.shutdown().await; +} + +#[tokio::test] +async fn invalid_stored_resource_name_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let config = config(&temp, &pki, &server); + rewrite_credential(&temp, |credential| { + credential.name = format!("organizations/{ORGANIZATION_UID}/clusters/not-a-uuid/clusterDevices/{DEVICE_UID}"); + }); + + assert_credential_failure(config, &server, "wrong device identity").await; +} + +#[tokio::test] +async fn invalid_stored_protocol_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let config = config(&temp, &pki, &server); + rewrite_credential(&temp, |credential| credential.protocol_version = "v2".to_owned()); + + assert_credential_failure(config, &server, "wrong device identity").await; +} + +#[tokio::test] +async fn stored_certificate_key_mismatch_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let now = OffsetDateTime::now_utc(); + let (identity_store, credential_store) = + pki.stores_with_certificate(&temp, now - time::Duration::hours(1), now + time::Duration::hours(23), false); + let config = config_with_stores(&temp, &pki, &server, identity_store, credential_store); + + assert_credential_failure(config, &server, "different device key").await; +} + +#[tokio::test] +async fn expired_stored_certificate_is_rejected_before_network_delivery() { + let pki = TestPki::new(); + let server = server(&pki, vec![]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let now = OffsetDateTime::now_utc(); + let (identity_store, credential_store) = + pki.stores_with_certificate(&temp, now - time::Duration::days(2), now - time::Duration::days(1), true); + let config = config_with_stores(&temp, &pki, &server, identity_store, credential_store); + + assert_credential_failure(config, &server, "not currently valid").await; +} + +#[tokio::test] +async fn sends_only_l0_fields_and_accepts_additive_response_fields() { + let pki = TestPki::new(); + let server = server(&pki, vec![Reply::ok("2038-01-19T03:14:07Z")]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + + assert_eq!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::Online { .. })).await, + HeartbeatStatus::Online { + server_time: "2038-01-19T03:14:07Z".to_owned() + } + ); + runtime.shutdown().await; + let seen = server.seen.lock().expect("seen lock"); + let request = &seen[0]; + let mut keys = request + .as_object() + .expect("heartbeat object") + .keys() + .map(String::as_str) + .collect::>(); + keys.sort_unstable(); + assert_eq!( + keys, + [ + "agentVersion", + "capabilities", + "clientTime", + "coarseNodeSummary", + "protocolVersion", + "requestId", + "sequence" + ] + ); + assert_eq!(request["capabilities"], json!(["heartbeat"])); + assert_eq!(request["coarseNodeSummary"], json!({"total": 8, "healthy": 7, "degraded": 1})); + assert_ne!(request["clientTime"], "2038-01-19T03:14:07Z"); + assert!(request.get("authorization").is_none()); +} + +#[tokio::test] +async fn restart_replays_pending_request_then_advances_sequence() { + let pki = TestPki::new(); + let first_server = server(&pki, vec![Reply::error(StatusCode::SERVICE_UNAVAILABLE)]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let first_config = config(&temp, &pki, &first_server); + let runtime = spawn_heartbeat_runtime(Some(first_config.clone()), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await; + runtime.shutdown().await; + let first = first_server.seen.lock().expect("seen lock")[0].clone(); + drop(first_server); + + let second_server = server(&pki, vec![Reply::ok("2026-08-22T01:02:03Z"), Reply::ok("2026-08-22T01:02:04Z")]).await; + let mut second_config = first_config; + second_config.endpoint = second_server.endpoint.clone(); + let runtime = spawn_heartbeat_runtime(Some(second_config), &shutdown, summary) + .expect("restart runtime") + .expect("configured runtime"); + tokio::time::timeout(Duration::from_secs(3), async { + while second_server.seen.lock().expect("seen lock").len() < 2 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("two heartbeats"); + runtime.shutdown().await; + + let seen = second_server.seen.lock().expect("seen lock"); + assert_eq!(seen[0]["requestId"], first["requestId"]); + assert_eq!(seen[0]["sequence"], first["sequence"]); + assert_ne!(seen[1]["requestId"], seen[0]["requestId"]); + assert_eq!(seen[1]["sequence"].as_u64(), seen[0]["sequence"].as_u64().map(|value| value + 1)); +} + +#[tokio::test] +async fn retry_after_is_respected_with_the_local_upper_bound() { + let pki = TestPki::new(); + let mut reply = Reply::error(StatusCode::TOO_MANY_REQUESTS); + reply.retry_after = Some("300"); + let server = server(&pki, vec![reply]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + assert_eq!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::BackingOff { .. })).await, + HeartbeatStatus::BackingOff { + delay: Duration::from_millis(80) + } + ); + runtime.shutdown().await; +} + +#[tokio::test] +async fn disconnects_use_exponential_backoff_with_a_cap() { + let pki = TestPki::new(); + let server = server( + &pki, + vec![ + Reply::error(StatusCode::SERVICE_UNAVAILABLE), + Reply::error(StatusCode::SERVICE_UNAVAILABLE), + Reply::error(StatusCode::SERVICE_UNAVAILABLE), + ], + ) + .await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + for delay in [20, 40, 80] { + assert_eq!( + wait_for(&mut status, |status| { + matches!(status, HeartbeatStatus::BackingOff { delay: observed } if *observed == Duration::from_millis(delay)) + }) + .await, + HeartbeatStatus::BackingOff { + delay: Duration::from_millis(delay) + } + ); + } + runtime.shutdown().await; +} + +#[tokio::test] +async fn revoked_credential_stops_and_exposes_local_status() { + let pki = TestPki::new(); + let mut reply = Reply::error(StatusCode::UNAUTHORIZED); + reply.body = json!({"details": [{"reason": "CREDENTIAL_REVOKED"}]}); + let server = server(&pki, vec![reply]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + let mut status = runtime.status(); + assert_eq!( + wait_for(&mut status, |status| matches!(status, HeartbeatStatus::AuthenticationStopped { .. })).await, + HeartbeatStatus::AuthenticationStopped { + status: 401, + reason: Some("CREDENTIAL_REVOKED".to_owned()) + } + ); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(server.seen.lock().expect("seen lock").len(), 1); + runtime.shutdown().await; +} + +#[tokio::test] +async fn shutdown_cancels_an_in_flight_request() { + let pki = TestPki::new(); + let mut reply = Reply::ok("2026-08-22T01:02:03Z"); + reply.delay = Duration::from_secs(5); + let server = server(&pki, vec![reply]).await; + let temp = tempfile::tempdir().expect("tempdir"); + let shutdown = CancellationToken::new(); + let runtime = spawn_heartbeat_runtime(Some(config(&temp, &pki, &server)), &shutdown, summary) + .expect("start runtime") + .expect("configured runtime"); + tokio::time::timeout(Duration::from_secs(3), async { + while server.seen.lock().expect("seen lock").is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .expect("request reached server"); + tokio::time::timeout(Duration::from_millis(250), runtime.shutdown()) + .await + .expect("cancellable shutdown"); +} + +#[test] +fn consumes_the_frozen_heartbeat_fixtures() { + let registry: Value = + serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/fixture-sets.json")).expect("fixture registry"); + let heartbeat = registry["sets"] + .as_array() + .expect("fixture sets") + .iter() + .find(|set| set["name"] == "heartbeat") + .expect("heartbeat fixture set"); + assert_eq!(heartbeat["status"], "populated"); + let valid: Value = + serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/heartbeat/valid.json")).expect("valid fixture"); + assert_eq!(valid["request"]["protocolVersion"], "v1"); + let overflow: Value = + serde_json::from_str(include_str!("../../protocol/agent/v1/fixtures/heartbeat/overflow.json")).expect("overflow fixture"); + assert_eq!(overflow["expected"]["httpStatus"], 422); +} + +#[cfg(unix)] +fn private_mode(path: &Path) { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(path, fs::Permissions::from_mode(0o600)).expect("private mode"); +} + +#[cfg(not(unix))] +fn private_mode(_path: &Path) {} diff --git a/scripts/check_scheduled_validation_freshness.py b/scripts/check_scheduled_validation_freshness.py new file mode 100644 index 000000000..d9bdf0597 --- /dev/null +++ b/scripts/check_scheduled_validation_freshness.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Fail when a critical scheduled validation has not started recently.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import json +import os +from pathlib import Path +import re +import sys +import tempfile +import unittest +from unittest import mock +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_validations(path: Path) -> list[tuple[str, int]]: + data = json.loads(path.read_text()) + if not isinstance(data, list) or not data: + raise ValueError("scheduled validation config must be a non-empty list") + + validations: list[tuple[str, int]] = [] + seen: set[str] = set() + for item in data: + if not isinstance(item, dict): + raise ValueError("scheduled validation entries must be objects") + workflow = item.get("workflow") + max_age_hours = item.get("max_age_hours") + if not isinstance(workflow, str) or not re.fullmatch( + r"\.github/workflows/[a-z0-9-]+\.yml", workflow + ): + raise ValueError(f"invalid scheduled validation workflow: {workflow!r}") + if workflow in seen: + raise ValueError(f"duplicate scheduled validation workflow: {workflow}") + if ( + not isinstance(max_age_hours, int) + or isinstance(max_age_hours, bool) + or max_age_hours <= 0 + ): + raise ValueError(f"invalid max_age_hours for {workflow}: {max_age_hours!r}") + seen.add(workflow) + validations.append((workflow, max_age_hours)) + return validations + + +def parse_timestamp(value: object) -> datetime: + if not isinstance(value, str): + raise ValueError(f"invalid run timestamp: {value!r}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError(f"run timestamp has no timezone: {value!r}") + return parsed.astimezone(timezone.utc) + + +def stale_reason( + run: dict[str, object] | None, now: datetime, max_age_hours: int +) -> str | None: + if run is None: + return "no scheduled run has been recorded" + created_at = parse_timestamp(run.get("created_at")) + age = now - created_at + if age > timedelta(hours=max_age_hours): + return f"last scheduled run is {age.total_seconds() / 3600:.1f}h old" + return None + + +def fetch_latest_scheduled_run( + repository: str, workflow: str, token: str, api_url: str +) -> dict[str, object] | None: + owner, repo = repository.split("/", 1) + workflow_name = Path(workflow).name + endpoint = ( + f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}" + f"/actions/workflows/{quote(workflow_name, safe='')}/runs?" + + urlencode({"event": "schedule", "per_page": 1}) + ) + request = Request( + endpoint, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urlopen(request, timeout=30) as response: + payload = json.load(response) + runs = payload.get("workflow_runs") + if not isinstance(runs, list): + raise ValueError(f"GitHub returned no workflow_runs list for {workflow}") + if not runs: + return None + if not isinstance(runs[0], dict): + raise ValueError(f"GitHub returned an invalid workflow run for {workflow}") + return runs[0] + + +def write_report(path: Path, failures: list[tuple[str, int, str, str]]) -> None: + lines = ["## Scheduled validation freshness"] + if not failures: + lines.append("") + lines.append("All critical scheduled validations have a recent scheduled run.") + else: + lines.extend( + [ + "", + "The following critical validations are stale or could not be inspected:", + "", + "| Workflow | Limit | Result | Last run |", + "| --- | ---: | --- | --- |", + ] + ) + for workflow, max_age_hours, reason, run_url in failures: + link = f"[open]({run_url})" if run_url else "—" + lines.append(f"| `{workflow}` | {max_age_hours}h | {reason} | {link} |") + path.write_text("\n".join(lines) + "\n") + + +def check_freshness( + config: Path, report: Path, repository: str, token: str, api_url: str +) -> int: + now = datetime.now(timezone.utc) + failures: list[tuple[str, int, str, str]] = [] + for workflow, max_age_hours in load_validations(config): + try: + run = fetch_latest_scheduled_run(repository, workflow, token, api_url) + reason = stale_reason(run, now, max_age_hours) + if reason is not None: + run_url = str(run.get("html_url", "")) if run else "" + failures.append((workflow, max_age_hours, reason, run_url)) + except Exception as error: + failures.append( + (workflow, max_age_hours, f"inspection failed: {error}", "") + ) + write_report(report, failures) + return 1 if failures else 0 + + +class SelfTests(unittest.TestCase): + NOW = datetime(2026, 8, 22, 12, tzinfo=timezone.utc) + + def test_freshness_boundaries(self) -> None: + at_limit = {"created_at": "2026-08-21T00:00:00Z"} + past_limit = {"created_at": "2026-08-20T23:59:59Z"} + self.assertIsNone(stale_reason(at_limit, self.NOW, 36)) + self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36)) + self.assertIsNotNone(stale_reason(None, self.NOW, 36)) + + def test_config_rejects_duplicate_and_invalid_entries(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "validations.json" + path.write_text( + json.dumps( + [ + {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}, + {"workflow": ".github/workflows/ci.yml", "max_age_hours": 0}, + ] + ) + ) + with self.assertRaises(ValueError): + load_validations(path) + path.write_text( + json.dumps( + [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 0}] + ) + ) + with self.assertRaises(ValueError): + load_validations(path) + + def test_check_reports_missing_runs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + config = root / "validations.json" + report = root / "report.md" + config.write_text( + json.dumps( + [ + {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}, + {"workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36}, + {"workflow": ".github/workflows/mint.yml", "max_age_hours": 36}, + ] + ) + ) + with mock.patch( + __name__ + ".fetch_latest_scheduled_run", + side_effect=[ + {"created_at": "2999-01-01T00:00:00Z"}, + None, + RuntimeError("API unavailable"), + ], + ): + self.assertEqual( + check_freshness( + config, + report, + "rustfs/rustfs", + "token", + "https://api.github.test", + ), + 1, + ) + contents = report.read_text() + self.assertIn(".github/workflows/fuzz.yml", contents) + self.assertIn("inspection failed: API unavailable", contents) + self.assertNotIn(".github/workflows/ci.yml`", contents) + + config.write_text( + json.dumps( + [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}] + ) + ) + with mock.patch( + __name__ + ".fetch_latest_scheduled_run", + return_value={"created_at": "2999-01-01T00:00:00Z"}, + ): + self.assertEqual( + check_freshness( + config, + report, + "rustfs/rustfs", + "token", + "https://api.github.test", + ), + 0, + ) + self.assertIn("All critical scheduled validations", report.read_text()) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", type=Path, default=ROOT / ".github/scheduled-validations.json" + ) + parser.add_argument("--report", type=Path) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + load_validations(args.config) + suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) + return ( + 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 + ) + if args.report is None: + parser.error("--report is required unless --self-test is used") + + repository = os.environ.get("GITHUB_REPOSITORY", "") + token = os.environ.get("GH_TOKEN", "") + api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com") + if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository): + parser.error("GITHUB_REPOSITORY must be owner/repository") + if not token: + parser.error("GH_TOKEN is required") + return check_freshness(args.config, args.report, repository, token, api_url) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 4edd41f0d..aa97dc944 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -10,11 +10,17 @@ import sys import tempfile import tomllib import unittest +from datetime import datetime, timezone from unittest import mock from pathlib import Path +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError ROOT = Path(__file__).resolve().parents[1] +SCHEDULED_ALERT_WORKFLOWS = tuple( + item["workflow"] + for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text()) +) def words(value: str) -> set[str]: @@ -252,6 +258,292 @@ def check_profile_definitions(root: Path) -> list[str]: return errors +def yaml_block(lines: list[str], key: str, indent: int) -> list[str] | None: + try: + start = lines.index(f"{' ' * indent}{key}:") + 1 + except ValueError: + return None + end = next( + ( + index + for index in range(start, len(lines)) + if lines[index].strip() + and not lines[index].lstrip().startswith("#") + and len(lines[index]) - len(lines[index].lstrip()) <= indent + ), + len(lines), + ) + return lines[start:end] + + +def workflow_step_block(job_lines: list[str], action: str) -> tuple[int, list[str]] | None: + uses_index = next( + ( + index + for index, line in enumerate(job_lines) + if ( + line.split("#", 1)[0].strip() == f"- uses: {action}" + and len(line) - len(line.lstrip()) == 6 + ) + or ( + line.split("#", 1)[0].strip() == f"uses: {action}" + and len(line) - len(line.lstrip()) == 8 + ) + ), + None, + ) + if uses_index is None: + return None + start = next( + ( + index + for index in range(uses_index, -1, -1) + if job_lines[index].lstrip().startswith("- ") + ), + uses_index, + ) + indent = len(job_lines[start]) - len(job_lines[start].lstrip()) + end = next( + ( + index + for index in range(start + 1, len(job_lines)) + if len(job_lines[index]) - len(job_lines[index].lstrip()) == indent + and job_lines[index].lstrip().startswith("- ") + ), + len(job_lines), + ) + return start, job_lines[start:end] + + +def alert_step_errors( + job_lines: list[str], + expected_action_if: str | None, + required_permissions: tuple[str, ...], + required_action_tokens: tuple[str, ...], +) -> list[str]: + checkout = workflow_step_block(job_lines, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0") + action = workflow_step_block(job_lines, "./.github/actions/schedule-failure-issue") + errors: list[str] = [] + permissions = yaml_block(job_lines, "permissions", 4) + permission_text = "\n".join(line.split("#", 1)[0] for line in permissions or []) + missing_permissions = [token for token in required_permissions if token not in permission_text] + if missing_permissions: + errors.append("alert job permissions missing " + ", ".join(missing_permissions)) + if checkout is None: + errors.append("checkout step is missing") + if action is None: + errors.append("local alert action step is missing") + if checkout is None or action is None: + return errors + + if checkout[0] >= action[0]: + errors.append("checkout must run before the local alert action") + checkout_ifs = [line.strip() for line in checkout[1] if line.strip().startswith("if:")] + if checkout_ifs: + errors.append("checkout step must not be conditional") + action_ifs = [line.strip() for line in action[1] if line.strip().startswith("if:")] + expected_ifs = [] if expected_action_if is None else [expected_action_if] + if action_ifs != expected_ifs: + errors.append("alert action has an invalid step condition") + action_text = "\n".join(line.split("#", 1)[0] for line in action[1]) + missing_action_tokens = [token for token in required_action_tokens if token not in action_text] + if missing_action_tokens: + errors.append("alert action inputs missing " + ", ".join(missing_action_tokens)) + return errors + + +def schedule_utc_slots(hour: int, minute: int, timezone_name: str | None) -> set[tuple[int, int]]: + if timezone_name is None: + return {(hour, minute)} + zone = ZoneInfo(timezone_name) + return { + (utc.hour, utc.minute) + for year in (2025, 2026) + for month in range(1, 13) + for utc in [datetime(year, month, 1, hour, minute, tzinfo=zone).astimezone(timezone.utc)] + } + + +def check_scheduled_alerts(root: Path) -> list[str]: + errors: list[str] = [] + schedule_slots: dict[tuple[int, int], list[str]] = {} + for relative in SCHEDULED_ALERT_WORKFLOWS: + path = root / relative + try: + lines = path.read_text().splitlines() + except FileNotFoundError: + errors.append(f"{relative}: missing scheduled validation workflow") + continue + + on_block = yaml_block(lines, "on", 0) + schedule_block = yaml_block(on_block or [], "schedule", 2) + schedule_lines = schedule_block or [] + cron_indices = [index for index, line in enumerate(schedule_lines) if re.match(r"^\s*-\s+cron:", line)] + if not cron_indices: + errors.append(f"{relative}: missing simple numeric schedule") + else: + for position, cron_index in enumerate(cron_indices): + cron_line = schedule_lines[cron_index] + schedule = re.match(r"^\s*-\s+cron:\s*[\"']?(\d+)\s+(\d+)\s+", cron_line) + if not schedule: + errors.append(f"{relative}: missing simple numeric schedule") + continue + minute, hour = map(int, schedule.groups()) + if minute == 0: + errors.append(f"{relative}: scheduled validation must avoid minute zero") + entry_end = cron_indices[position + 1] if position + 1 < len(cron_indices) else len(schedule_lines) + entry = "\n".join(schedule_lines[cron_index + 1 : entry_end]) + timezone_match = re.search(r"^\s*timezone:\s*[\"']?([^\"'\s]+)", entry, re.MULTILINE) + timezone_name = timezone_match.group(1) if timezone_match else None + try: + utc_slots = schedule_utc_slots(hour, minute, timezone_name) + except ZoneInfoNotFoundError: + errors.append(f"{relative}: unknown schedule timezone {timezone_name}") + continue + for slot in utc_slots: + schedule_slots.setdefault(slot, []).append(relative) + + job_lines = yaml_block(lines, "alert-on-failure", 2) + if job_lines is None: + errors.append(f"{relative}: missing alert-on-failure job") + continue + job = "\n".join(line.split("#", 1)[0] for line in job_lines) + required = ( + "always()", + "github.event_name == 'schedule'", + "contains(needs.*.result, 'failure')", + "issues: write", + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: ./.github/actions/schedule-failure-issue", + "github-token: ${{ secrets.GITHUB_TOKEN }}", + ) + missing = [token for token in required if token not in job] + if missing: + errors.append(f"{relative}: alert-on-failure missing {', '.join(missing)}") + else: + errors.extend( + f"{relative}: {error}" + for error in alert_step_errors(job_lines, None, ("issues: write",), ("github-token: ${{ secrets.GITHUB_TOKEN }}",)) + ) + + for (hour, minute), workflows in schedule_slots.items(): + if len(workflows) > 1: + errors.append( + f"scheduled validations share {hour:02d}:{minute:02d} UTC: {', '.join(workflows)}" + ) + + watchdog_path = root / ".github/workflows/scheduled-validation-watchdog.yml" + try: + watchdog_lines = watchdog_path.read_text().splitlines() + except FileNotFoundError: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing completion watchdog") + return errors + watchdog_on = yaml_block(watchdog_lines, "on", 0) + watchdog_run = yaml_block(watchdog_on or [], "workflow_run", 2) + watchdog_workflows = yaml_block(watchdog_run or [], "workflows", 4) + if watchdog_workflows is None: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing workflow_run workflows") + return errors + watchdog_sources = "\n".join(line.split("#", 1)[0] for line in watchdog_workflows) + for relative in SCHEDULED_ALERT_WORKFLOWS: + path = root / relative + if not path.is_file(): + continue + source = path.read_text() + match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE) + if not match: + errors.append(f"{relative}: missing workflow name") + elif f'- "{match.group(1).strip()}"' not in watchdog_sources: + errors.append(f"{relative}: missing from scheduled completion watchdog") + watchdog_job_lines = yaml_block(watchdog_lines, "alert-on-incomplete-run", 2) + if watchdog_job_lines is None: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing alert-on-incomplete-run job") + return errors + watchdog_job = "\n".join(line.split("#", 1)[0] for line in watchdog_job_lines) + required = ( + "github.event.workflow_run.event == 'schedule'", + "github.event.workflow_run.conclusion != 'success'", + "github.event.workflow_run.conclusion != 'failure'", + "actions: read", + "issues: write", + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: ./.github/actions/schedule-failure-issue", + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "workflow-name: ${{ github.event.workflow_run.name }}", + "source-run-id: ${{ github.event.workflow_run.id }}", + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}", + "source-event: ${{ github.event.workflow_run.event }}", + "source-ref-name: ${{ github.event.workflow_run.head_branch }}", + "source-sha: ${{ github.event.workflow_run.head_sha }}", + ) + missing = [token for token in required if token not in watchdog_job] + if missing: + errors.append( + ".github/workflows/scheduled-validation-watchdog.yml: missing " + ", ".join(missing) + ) + else: + errors.extend( + ".github/workflows/scheduled-validation-watchdog.yml: " + error + for error in alert_step_errors( + watchdog_job_lines, + None, + ("actions: read", "issues: write"), + ( + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "workflow-name: ${{ github.event.workflow_run.name }}", + "source-run-id: ${{ github.event.workflow_run.id }}", + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}", + "source-event: ${{ github.event.workflow_run.event }}", + "source-ref-name: ${{ github.event.workflow_run.head_branch }}", + "source-sha: ${{ github.event.workflow_run.head_sha }}", + ), + ) + ) + + freshness_path = root / ".github/workflows/scheduled-validation-freshness.yml" + try: + freshness_lines = freshness_path.read_text().splitlines() + except FileNotFoundError: + errors.append(".github/workflows/scheduled-validation-freshness.yml: missing freshness check") + return errors + freshness_job_lines = yaml_block(freshness_lines, "check-freshness", 2) + if freshness_job_lines is None: + errors.append(".github/workflows/scheduled-validation-freshness.yml: missing check-freshness job") + return errors + freshness_job = "\n".join(line.split("#", 1)[0] for line in freshness_job_lines) + required = ( + "python3 scripts/check_scheduled_validation_freshness.py", + "actions: read", + "issues: write", + "if: failure()", + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: ./.github/actions/schedule-failure-issue", + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md", + ) + missing = [token for token in required if token not in freshness_job] + if missing: + errors.append( + ".github/workflows/scheduled-validation-freshness.yml: missing " + ", ".join(missing) + ) + else: + errors.extend( + ".github/workflows/scheduled-validation-freshness.yml: " + error + for error in alert_step_errors( + freshness_job_lines, + "if: failure()", + ("actions: read", "issues: write"), + ( + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md", + ), + ) + ) + if not (root / "scripts/check_scheduled_validation_freshness.py").is_file(): + errors.append("scripts/check_scheduled_validation_freshness.py: missing freshness checker") + return errors + + def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]: try: expected_digest = profile_selection(root, profile) @@ -281,6 +573,7 @@ def validate(root: Path) -> list[str]: errors.extend(check_runner_selection(root)) errors.extend(check_s3_tests_runner(root)) errors.extend(check_profile_definitions(root)) + errors.extend(check_scheduled_alerts(root)) return errors @@ -363,6 +656,7 @@ class SelfTests(unittest.TestCase): mock.patch(__name__ + ".check_fuzz_targets", return_value=[]), mock.patch(__name__ + ".check_runner_selection", return_value=[]), mock.patch(__name__ + ".check_profile_definitions", return_value=[]), + mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]), ): self.assertEqual(len(validate(root)), 1) @@ -413,6 +707,272 @@ class SelfTests(unittest.TestCase): with mock.patch.object(sys, "platform", "linux"): self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1) + def test_scheduled_alerts_require_completion_watchdog(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + alert = ( + " alert-on-failure:\n" + " if: always() && github.event_name == 'schedule' && " + "contains(needs.*.result, 'failure')\n" + " permissions:\n" + " issues: write\n" + " steps:\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " - uses: ./.github/actions/schedule-failure-issue\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + ) + names: list[str] = [] + for index, relative in enumerate(SCHEDULED_ALERT_WORKFLOWS, start=1): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + names.append(path.stem) + path.write_text( + f'name: "{path.stem}"\n' + f'on:\n schedule:\n - cron: "{index} {index} * * *"\n' + f'jobs:\n{alert}' + ) + watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml" + watchdog.write_text( + "on:\n workflow_run:\n workflows:\n" + + "\n".join(f' - "{name}"' for name in names) + + "\njobs:\n" + + " alert-on-incomplete-run:\n" + + " github.event.workflow_run.event == 'schedule'\n" + + " github.event.workflow_run.conclusion != 'success'\n" + + " github.event.workflow_run.conclusion != 'failure'\n" + + " permissions:\n" + + " actions: read\n" + + " issues: write\n" + + " steps:\n" + + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + + " - uses: ./.github/actions/schedule-failure-issue\n" + + " with:\n" + + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + + " workflow-name: ${{ github.event.workflow_run.name }}\n" + + " source-run-id: ${{ github.event.workflow_run.id }}\n" + + " source-run-attempt: ${{ github.event.workflow_run.run_attempt }}\n" + + " source-event: ${{ github.event.workflow_run.event }}\n" + + " source-ref-name: ${{ github.event.workflow_run.head_branch }}\n" + + " source-sha: ${{ github.event.workflow_run.head_sha }}\n" + ) + freshness = root / ".github/workflows/scheduled-validation-freshness.yml" + freshness.write_text( + "jobs:\n" + " check-freshness:\n" + " permissions:\n" + " actions: read\n" + " issues: write\n" + " steps:\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " - run: python3 scripts/check_scheduled_validation_freshness.py\n" + " - uses: ./.github/actions/schedule-failure-issue\n" + " if: failure()\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + " details-file: ${{ runner.temp }}/scheduled-validation-freshness.md\n" + ) + checker = root / "scripts/check_scheduled_validation_freshness.py" + checker.parent.mkdir() + checker.write_text("") + self.assertEqual(check_scheduled_alerts(root), []) + + first = root / SCHEDULED_ALERT_WORKFLOWS[0] + mutations = ( + ("contains(needs.*.result, 'failure')", "false"), + ("issues: write", "issues: read"), + ( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: actions/checkout@missing", + ), + ( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ( + " - uses: ./.github/actions/schedule-failure-issue\n", + " - uses: ./.github/actions/schedule-failure-issue\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"), + ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"), + ) + for required, replacement in mutations: + original = first.read_text() + first.write_text(original.replace(required, replacement)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(original) + + first_original = first.read_text() + real_steps = ( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " - uses: ./.github/actions/schedule-failure-issue\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + ) + first.write_text( + first_original.replace( + real_steps, + " - run: |\n" + " : <<'MARKER'\n" + " uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " MARKER\n" + " - run: |\n" + " : <<'MARKER'\n" + " uses: ./.github/actions/schedule-failure-issue\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + " MARKER\n", + ) + ) + self.assertTrue(check_scheduled_alerts(root)) + first.write_text( + first_original.replace( + real_steps, + " - uses: ./.github/actions/schedule-failure-issue\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + + watchdog_mutations = ( + ("actions: read", "actions: none"), + ("issues: write", "issues: read"), + ( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: actions/checkout@missing", + ), + ( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ( + " - uses: ./.github/actions/schedule-failure-issue\n", + " - uses: ./.github/actions/schedule-failure-issue\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"), + ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"), + ("source-event: ${{ github.event.workflow_run.event }}", "source-event: watchdog"), + ( + "source-ref-name: ${{ github.event.workflow_run.head_branch }}", + "source-ref-name: main", + ), + ("source-sha: ${{ github.event.workflow_run.head_sha }}", "source-sha: missing"), + ) + for required, replacement in watchdog_mutations: + original = watchdog.read_text() + watchdog.write_text(original.replace(required, replacement)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(original) + + watchdog_original = watchdog.read_text() + watchdog.write_text( + watchdog_original.replace("issues: write", "issues: read") + + " decoy:\n permissions:\n issues: write\n" + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(watchdog_original) + + first_original = first.read_text() + first.write_text( + first_original.replace(' schedule:\n - cron: "1 1 * * *"\n', "") + + ' decoy:\n strategy:\n matrix:\n cron:\n - "1 1 * * *"\n' + + ' runs-on: ubuntu-latest\n steps:\n - run: true\n' + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + + first.write_text( + first_original.replace( + ' - cron: "1 1 * * *"\n', + ' - cron: "1 1 * * *"\n - cron: "0 5 * * *"\n', + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text( + first_original.replace( + ' - cron: "1 1 * * *"\n', + ' - cron: "1 1 * * *"\n - cron: "2 2 * * *"\n', + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + + watchdog.write_text( + watchdog_original.replace(f' - "{names[0]}"\n', "") + + f' decoy:\n strategy:\n matrix:\n workflow:\n - "{names[0]}"\n' + + ' runs-on: ubuntu-latest\n steps:\n - run: true\n' + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(watchdog_original) + + watchdog.write_text(watchdog_original.replace(f' - "{names[0]}"\n', "")) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(watchdog_original) + original = first.read_text() + first.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "0 0', original, count=1)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(original) + + second = root / SCHEDULED_ALERT_WORKFLOWS[1] + second_original = second.read_text() + second.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "1 1', second_original, count=1)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + second.write_text(second_original) + + first.write_text( + first_original.replace( + ' - cron: "1 1 * * *"\n', + ' - cron: "7 0 * * *"\n timezone: "Asia/Shanghai"\n', + ) + ) + second.write_text( + second_original.replace( + ' - cron: "2 2 * * *"\n', + ' - cron: "2 2 * * *"\n - cron: "7 16 * * *"\n', + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + second.write_text(second_original) + + freshness_original = freshness.read_text() + freshness.write_text(freshness_original.replace("details-file:", "report-file:")) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing") + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: actions/checkout@missing", + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " if: github.event_name == 'workflow_dispatch'\n", + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace("if: failure()", "if: github.event_name == 'workflow_dispatch'") + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace("issues: write", "issues: read") + + " decoy:\n permissions:\n issues: write\n" + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) @@ -436,7 +996,7 @@ def main() -> int: for error in errors: print(f"ERROR: {error}", file=sys.stderr) return 1 - print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired") + print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired") return 0 diff --git a/scripts/security/check_cache_save_if.sh b/scripts/security/check_cache_save_if.sh index 3e63bdc8a..27abc6545 100755 --- a/scripts/security/check_cache_save_if.sh +++ b/scripts/security/check_cache_save_if.sh @@ -64,3 +64,25 @@ if [ "$status" -ne 0 ]; then fi echo "OK: every ./.github/actions/setup call states cache-save-if explicitly" + +# rust-cache hashes every CARGO*, CC*, CFLAGS*, CXX*, CMAKE*, and RUST* +# variable that is present when the setup action runs. The dedicated writer +# and the CI readers therefore need identical workflow-level compiler env. +compiler_env() { + awk ' + /^env:[[:space:]]*$/ { in_env = 1; next } + in_env && /^[^[:space:]]/ { exit } + in_env && /^ (CARGO|CC|CFLAGS|CXX|CMAKE|RUST)[A-Z0-9_]*:/ { print } + ' "$1" | sort +} + +ci_env="$(compiler_env .github/workflows/ci.yml)" +warm_env="$(compiler_env .github/workflows/cache-warm.yml)" + +if [ "$ci_env" != "$warm_env" ]; then + echo "CI and cache-warm compiler environments differ; rust-cache keys will not match:" >&2 + diff -u <(printf '%s\n' "$ci_env") <(printf '%s\n' "$warm_env") >&2 || true + exit 1 +fi + +echo "OK: cache-warm and CI compiler environments match" diff --git a/scripts/security/check_performance_ab_workflow.sh b/scripts/security/check_performance_ab_workflow.sh index 5465e4fdf..09b97cc0d 100755 --- a/scripts/security/check_performance_ab_workflow.sh +++ b/scripts/security/check_performance_ab_workflow.sh @@ -13,8 +13,29 @@ require_absent_pattern() { fi } +require_present_pattern() { + local pattern="$1" + local description="$2" + + if ! grep -Eq -- "$pattern" "$workflow"; then + echo "invalid performance A/B workflow contract: $description" >&2 + exit 1 + fi +} + require_absent_pattern '(^|[^[:alnum:]_])pull_request(_target)?([^[:alnum:]_]|$)' "the workflow must not contain PR event handling" require_absent_pattern 'pull-requests[[:space:]]*:[[:space:]]*write' "the workflow must not receive PR write permission" require_absent_pattern 'permissions[[:space:]]*:[[:space:]]*write-all' "the workflow must not receive broad write permission" +require_absent_pattern '^[[:space:]]*push:' "the workflow must not spend a release build on every main push" +require_present_pattern 'listWorkflowRuns' "the scheduled baseline must come from workflow history" +require_present_pattern 'status:[[:space:]]*"success"' "the scheduled baseline must be a successful run" +require_present_pattern 'SCHEDULED_BASELINE_SHA' "the resolved scheduled baseline must reach the comparison" +require_present_pattern "SCHEDULED_BASELINE_SHA:-\\\$candidate_sha" "the first scheduled run must seed from its verified candidate" +require_present_pattern 'git merge-base --is-ancestor' "the scheduled baseline must stay on candidate history" +require_present_pattern 'Cache successful candidate baseline' "a successful candidate must become the next cached baseline" +if ! sed -n '/^ warp-ab:/,/^ alert-on-failure:/p' "$workflow" | grep -Eq '^ timeout-minutes:[[:space:]]*180([[:space:]]|$)'; then + echo "invalid performance A/B workflow contract: the cold-cache path must fit both builds, the A/B run, and evidence publication" >&2 + exit 1 +fi -echo "Performance A/B workflow trust boundary ok." +echo "Performance A/B workflow contract ok."