mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-01 16:50:45 +00:00
Compare commits
38 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ec218bfecb | |||
| a5c6f6195e | |||
| ec0208480e | |||
| 62b6b6149c | |||
| 741de6f81c | |||
| f5ff8783f4 | |||
| 12ae143bee | |||
| 628663f468 | |||
| d457378c02 | |||
| e76cb67e93 | |||
| 8c8c74cbc3 | |||
| 9d85d21ee6 | |||
| 4e9b3818eb | |||
| f0b584d61a | |||
| d0fec44808 | |||
| b4f58b81cf | |||
| 7682a12b58 | |||
| 65f30bd5c1 | |||
| 7462c25319 | |||
| 1c663b8207 | |||
| cb2dfec17e | |||
| c1f906d128 | |||
| b08331335a | |||
| aae5cc4919 | |||
| 69e987b4de | |||
| 7a32ca2adb | |||
| d7156290d5 | |||
| 6ec1942ba3 | |||
| 606d8a3a8a | |||
| 2c2c1133d4 | |||
| 8a96fedfdd | |||
| ce00d8f7e5 | |||
| cbd53974f9 | |||
| 65ea7db6a9 | |||
| bdc00c8d0d | |||
| 84f7ab1f5c | |||
| 8b75145a29 | |||
| c9f88cff66 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=1520cd3027c3a6924afc6a204eddd5475bc30f0109d4c2847456a66177263244
|
||||
sha256-linux=26155b7f1c5bf6bf596ff24a4136e6767ea4ee6461c32db7fbe1400791da2c28
|
||||
sha256-darwin=419d60fc0a4451248ae91c2dd185dcb261f4bb81a9f26ba7f357a6c74d650892
|
||||
sha256-linux=96dffebaebb2c100d71490beabac5faf2a4d9b4ab7cbab08e4ec359ee23b77cd
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=ba8f392b617cf9f2338142233a417e43927b28096635e628233ce8a60cf49d10
|
||||
sha256=ce16185f87c6cd30aa0891ebd22be3352aedc10040d48cc5b00652d72f7be3b5
|
||||
|
||||
@@ -605,6 +605,7 @@ default-filter = """
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
| test(/^reliant::tiering::/)
|
||||
| test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
|
||||
| test(/^on_demand_migration::(get_basic_test::(get_miss_pulls_inline_and_serves_locally_afterwards|head_miss_answers_from_the_source_without_persisting)|interaction_test::test_odm_admin_config_is_redacted_and_status_counts_match_the_source)$/)
|
||||
|
||||
@@ -46,9 +46,6 @@ concurrency:
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
UPGRADE_SOURCE_VERSION: 1.0.0-rc.5
|
||||
UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip
|
||||
UPGRADE_SOURCE_SHA256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d
|
||||
|
||||
jobs:
|
||||
build:
|
||||
@@ -71,14 +68,6 @@ jobs:
|
||||
install-build-packaging-tools: "false"
|
||||
install-test-tools: "false"
|
||||
|
||||
- name: Download pinned previous release
|
||||
run: |
|
||||
mkdir -p target/debug
|
||||
archive="target/debug/$UPGRADE_SOURCE_ASSET"
|
||||
curl --fail --location --retry 3 --output "$archive" \
|
||||
"https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
|
||||
echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict
|
||||
|
||||
- name: Build current RustFS binary
|
||||
run: python3 scripts/e2e_binary.py build
|
||||
|
||||
@@ -89,19 +78,31 @@ jobs:
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e.json
|
||||
target/debug/${{ env.UPGRADE_SOURCE_ASSET }}
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
compression-level: 0
|
||||
overwrite: true
|
||||
|
||||
upgrade:
|
||||
name: Upgrade compatibility
|
||||
name: Upgrade compatibility (${{ matrix.source.version }})
|
||||
needs: build
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
source:
|
||||
- version: 1.0.0-rc.5
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip
|
||||
sha256: 3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d
|
||||
- version: 1.0.1
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
|
||||
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
|
||||
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
|
||||
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -132,6 +133,8 @@ jobs:
|
||||
set -euo pipefail
|
||||
mkdir -p "$SOURCE_DIR"
|
||||
archive="target/debug/$UPGRADE_SOURCE_ASSET"
|
||||
curl --fail --location --retry 3 --output "$archive" \
|
||||
"https://github.com/rustfs/rustfs/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}"
|
||||
echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
@@ -148,14 +151,18 @@ jobs:
|
||||
run: |
|
||||
export no_proxy="$NO_PROXY"
|
||||
cases=(
|
||||
direct-upgrade:direct_upgrade_from_rc2_preserves_object_contracts
|
||||
mixed-version-upgrade:rolling_upgrade_from_rc2_preserves_mixed_version_contracts
|
||||
direct-upgrade:direct_upgrade_from_previous_release_preserves_object_contracts
|
||||
mixed-version-upgrade:rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
bucket-config-upgrade:direct_upgrade_from_previous_release_preserves_bucket_configuration
|
||||
bucket-config-rollback:rollback_to_previous_release_reads_current_bucket_metadata
|
||||
odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration
|
||||
multipart-layout-upgrade:direct_upgrade_from_rc5_preserves_multipart_layouts
|
||||
multipart-layout-baseline:rc5_baseline_replicates_multipart_layouts
|
||||
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
|
||||
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
|
||||
)
|
||||
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
else
|
||||
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
|
||||
fi
|
||||
status=0
|
||||
for entry in "${cases[@]}"; do
|
||||
artifact="${entry%%:*}"
|
||||
@@ -180,7 +187,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: upgrade-g09-evidence-${{ github.run_number }}
|
||||
name: upgrade-g09-evidence-${{ matrix.source.version }}-${{ github.run_number }}
|
||||
path: ${{ runner.temp }}/rustfs-upgrade-g09-evidence
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
@@ -189,7 +196,7 @@ jobs:
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: upgrade-server-logs-${{ github.run_number }}
|
||||
name: upgrade-server-logs-${{ matrix.source.version }}-${{ github.run_number }}
|
||||
path: ${{ runner.temp }}/rustfs-upgrade-logs
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
@@ -6,29 +6,20 @@ name: Gateway Integration Sync
|
||||
# Merges `main` into the long-lived `gateway/integration` branch every weekday
|
||||
# so the s3s -> gateway migration (rustfs/backlog#2734, task T0.1 in #2736)
|
||||
# never accumulates merge debt. A clean merge lands as a `--no-ff` merge commit
|
||||
# pushed straight to the branch. A conflicting merge is aborted and surfaced as
|
||||
# a pull request from `sync/<source>-<date>`, whose head is the un-merged
|
||||
# source commit, so the conflict is resolved the same day, in review, with no
|
||||
# other change riding along. The run itself stays green in that case: the pull
|
||||
# request is the signal. Any other failure (push rejected, pull request not
|
||||
# created) fails the run and, on the schedule, files the shared failure issue.
|
||||
# pushed straight to the branch. A conflicting merge is aborted and fails the
|
||||
# run after its summary records the source and target SHAs, run URL, and exact
|
||||
# conflicting paths. It creates no sync branch or pull request. Resolve the
|
||||
# merge in an isolated worktree based on the latest target, preserve the
|
||||
# two-parent merge commit, and push only after the required checks pass and the
|
||||
# target still matches the merge's first parent. If the target moves, restart
|
||||
# from its new head.
|
||||
#
|
||||
# Resolving a conflict pull request: merge `origin/gateway/integration` into
|
||||
# the sync branch, resolve, push, get the review. Land it with a fast-forward
|
||||
# push of the resolved merge commit to `gateway/integration`, not with the
|
||||
# squash button: a squash drops the merge ancestry and the same hunks conflict
|
||||
# again on the next run.
|
||||
#
|
||||
# Write scope: only `gateway/integration` and `sync/*` are ever pushed, and
|
||||
# both push targets below are literal refs. The pushes use a one-hour
|
||||
# installation token of the organisation's own GitHub App (contents: write)
|
||||
# because that app is the only non-admin bypass actor of the
|
||||
# Write scope: only `gateway/integration` is ever pushed. The push uses a
|
||||
# one-hour installation token of the organisation's GitHub App (contents:
|
||||
# write) because that app is the only non-admin bypass actor of the
|
||||
# `gateway/integration` ruleset; GITHUB_TOKEN cannot be a ruleset bypass actor,
|
||||
# so it cannot push through the required reviews and checks. The conflict pull
|
||||
# request is opened with FLAKE_UPDATE_TOKEN, the PAT that already opens the
|
||||
# weekly flake.lock pull requests, because this repository keeps "Allow GitHub
|
||||
# Actions to create and approve pull requests" switched off. GITHUB_TOKEN is
|
||||
# therefore read-only here.
|
||||
# so it cannot push through the required reviews and checks. GITHUB_TOKEN is
|
||||
# read-only here.
|
||||
#
|
||||
# `schedule` fires only from the default branch, so the cron is live once this
|
||||
# file exists on `main`. `workflow_dispatch` runs from any ref that carries the
|
||||
@@ -94,6 +85,7 @@ jobs:
|
||||
id: merge
|
||||
env:
|
||||
SOURCE_REF: ${{ inputs.source_ref || 'main' }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! [[ "$SOURCE_REF" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] \
|
||||
@@ -119,13 +111,25 @@ jobs:
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
git diff --name-only --diff-filter=U > "$RUNNER_TEMP/conflicts.txt"
|
||||
git merge --abort
|
||||
if [[ ! -s "$RUNNER_TEMP/conflicts.txt" ]]; then
|
||||
conflicts=$(git diff --name-only --diff-filter=U)
|
||||
if [[ -z "$conflicts" ]]; then
|
||||
git merge --abort || true
|
||||
echo "::error::git merge failed without reporting conflicting paths; this is not a conflict"
|
||||
exit 1
|
||||
fi
|
||||
echo "result=conflict" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "## Gateway integration sync stopped on conflicts"
|
||||
echo
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Source: \`${SOURCE_REF}\` at ${source_sha}"
|
||||
echo "- Target: \`${TARGET_BRANCH}\` at ${target_sha}"
|
||||
echo
|
||||
echo "Conflicting files:"
|
||||
printf '%s\n' "$conflicts" | sed 's/^/- /'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
git merge --abort
|
||||
echo "::error::${SOURCE_REF} cannot be merged cleanly into ${TARGET_BRANCH}; no branch or pull request was created"
|
||||
exit 1
|
||||
|
||||
- name: Push the merge commit
|
||||
if: steps.merge.outputs.result == 'merged'
|
||||
@@ -137,50 +141,12 @@ jobs:
|
||||
git push origin HEAD:refs/heads/gateway/integration
|
||||
echo "Merged \`${SOURCE_REF}\` (${SOURCE_SHA}) into \`${TARGET_BRANCH}\` as $(git rev-parse HEAD)." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Open the conflict pull request
|
||||
if: steps.merge.outputs.result == 'conflict'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.FLAKE_UPDATE_TOKEN }}
|
||||
SOURCE_REF: ${{ steps.merge.outputs.source_ref }}
|
||||
SOURCE_SHA: ${{ steps.merge.outputs.source_sha }}
|
||||
TARGET_SHA: ${{ steps.merge.outputs.target_sha }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
date_utc=$(date -u +%Y-%m-%d)
|
||||
sync_branch="sync/${SOURCE_REF//\//-}-${date_utc}"
|
||||
git push origin "refs/remotes/origin/${SOURCE_REF}:refs/heads/${sync_branch}"
|
||||
{
|
||||
echo "The scheduled merge of \`${SOURCE_REF}\` into \`${TARGET_BRANCH}\` stopped on conflicts and was aborted; \`${TARGET_BRANCH}\` is unchanged."
|
||||
echo
|
||||
echo "This branch is the un-merged \`${SOURCE_REF}\` head. Merge \`origin/${TARGET_BRANCH}\` into it, resolve the files below, push, and get the review. Land the resolved merge commit with a fast-forward push to \`${TARGET_BRANCH}\`; a squash merge drops the merge ancestry and the same hunks conflict again on the next run. Do not include any change that is not conflict resolution."
|
||||
echo
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Source: \`${SOURCE_REF}\` at ${SOURCE_SHA}"
|
||||
echo "- Target: \`${TARGET_BRANCH}\` at ${TARGET_SHA}"
|
||||
echo
|
||||
echo "## Conflicting files"
|
||||
echo
|
||||
echo "Output of \`git diff --name-only --diff-filter=U\` captured before \`git merge --abort\`:"
|
||||
echo
|
||||
sed 's/^/- /' "$RUNNER_TEMP/conflicts.txt"
|
||||
} > "$RUNNER_TEMP/pr-body.md"
|
||||
title="sync: merge ${SOURCE_REF} ${date_utc} (conflicts)"
|
||||
existing=$(gh pr list --repo "$GITHUB_REPOSITORY" --base "$TARGET_BRANCH" --head "$sync_branch" --state open --json number --jq '.[0].number // empty')
|
||||
if [[ -n "$existing" ]]; then
|
||||
gh pr edit "$existing" --repo "$GITHUB_REPOSITORY" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md"
|
||||
pr_url=$(gh pr view "$existing" --repo "$GITHUB_REPOSITORY" --json url --jq .url)
|
||||
else
|
||||
pr_url=$(gh pr create --repo "$GITHUB_REPOSITORY" --base "$TARGET_BRANCH" --head "$sync_branch" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md")
|
||||
fi
|
||||
echo "Conflicts in \`${SOURCE_REF}\` -> \`${TARGET_BRANCH}\`; pull request: ${pr_url}" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
needs: [sync]
|
||||
# Same ci-8 mechanism audit.yml and coverage.yml use: a scheduled run that
|
||||
# could not push or could not open the conflict pull request must not stay
|
||||
# visible only in the Actions tab.
|
||||
# could not push or encountered merge conflicts must not stay visible only
|
||||
# in the Actions tab.
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
||||
@@ -21,6 +21,7 @@ on:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "crates/utils/src/dunce.rs"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/actions/setup/**"
|
||||
@@ -31,6 +32,7 @@ on:
|
||||
- "crates/ecstore/src/disk/**"
|
||||
- "crates/ecstore/src/store/init_format.rs"
|
||||
- "crates/ecstore/Cargo.toml"
|
||||
- "crates/utils/src/dunce.rs"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/actions/setup/**"
|
||||
@@ -77,6 +79,10 @@ jobs:
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib rename_all_ -- --nocapture
|
||||
|
||||
- name: Test extended-length path conversion
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-utils --no-default-features --features path --lib extended_length_path -- --nocapture
|
||||
|
||||
- name: Test Windows handle guards
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib windows_ -- --nocapture
|
||||
|
||||
@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **Presigned URLs honour only signed headers** (GHSA-g8w9-qw9q-fghr): a SigV4 presigned request that carries an `x-amz-*` request header not listed in `X-Amz-SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3. Previously the holder of a presigned `PutObject` URL could add unsigned `x-amz-tagging`, `x-amz-storage-class`, `x-amz-website-redirect-location`, ACL, metadata, Object Lock or SSE headers and have them applied. Presigners that intend a property must set it before signing so the SDK lists the header in `SignedHeaders`; `x-amz-cf-id` (CloudFront) remains tolerated unsigned. Header-signed SigV4 and SigV2 requests are unchanged.
|
||||
|
||||
### Fixed
|
||||
- **Replication status writebacks no longer invalidate scanner cycles**: every replication status writeback (`persist_replication_state_if_current` in `Update` mode, including the heals the scanner queues during its own walk) marked the bucket dirty on the namespace path, advanced the dirty-usage generation and woke the scanner, so a site-replication resync or any steadily replicated bucket ended every cycle `superseded`, the authoritative usage baseline was never certified, and the console reported usage freshness as stale for as long as replication ran (#8375). Those writebacks are now status class: they do not touch the dirty-usage generation, the activity snapshot, the dirty-usage wake or the durable dirty journal, and only the process-local producer coverage is recorded. They need no rescan queue, because a deployment with an active replication rule already walks every bucket on every cycle and that walk refreshes `replication_stats`. Tier-expiration cleanup, which removes data, stays on the namespace path. No persisted format, protocol, admin API or configuration changes. Design: `docs/architecture/scanner-per-bucket-convergence.md`.
|
||||
- **SSE-C TLS requirement now covers copy-source keys**: with `RUSTFS_SSE_C_REQUIRE_TLS=true`, a `CopyObject` or `UploadPartCopy` that carried only the `x-amz-copy-source-server-side-encryption-customer-*` headers (an SSE-C source copied to a destination that is not SSE-C) was accepted on a plaintext transport and was not counted in `rustfs_ssec_plaintext_requests_total`, although those headers carry the source object's customer key. They now count as SSE-C headers: such requests are counted, and refused with `400 InvalidRequest` when the switch is on. Requests over TLS are unaffected.
|
||||
- **SSE-KMS encryption context could replace the location entry**: data keys are wrapped under an encryption-context entry, keyed by the bucket name, that ties them to the object's location. A client `x-amz-server-side-encryption-context` entry under the same key replaced it, and because the client context is stored with the object, the replacement also applied on every read. SSE-KMS writes now refuse such a context with `400 InvalidArgument`; other keys, including the names of other buckets, are unaffected. Objects stored with such an entry by an earlier release remain readable. See `docs/operations/kms-backend-security.md`.
|
||||
- **Two admin handlers reported a missing policy or user as `500 InternalError`**: `GET /rustfs/admin/v3/info-canned-policy` and `DELETE /rustfs/admin/v3/remove-user` wrapped every IAM error as an internal fault instead of classifying it, so a lookup for something that does not exist answered `500 InternalError` where the rest of the admin API answers `404 NoSuchResource` — `GET /rustfs/admin/v3/user-info` already did, and `iam_error_to_s3_error` already mapped `NoSuchPolicy`, `NoSuchUser`, `NoSuchServiceAccount` and `NoSuchTempAccount` to `NoSuchResource`. A client could not tell "does not exist" from a real server failure, and a controller that checks existence before creating (a Kubernetes operator reconciling policies or users) could never create the object: the missing-policy lookup failed with a `500` and it retried forever. Both handlers now map their IAM errors through the shared `iam_error_to_s3_error` helper, so a missing policy or user is `404 NoSuchResource` and the `error` is preserved as the S3 error's source; the three IAM calls in the removal path (temporary-user lookup, service-account lookup, delete) are all reclassified, and every other IAM error stays `500 InternalError`. Fixes #8117.
|
||||
@@ -85,6 +86,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Four-layer regression-prevention tests guard against silent feature deletion: compile-time module assertion, module-presence unit test, cross-module `Protocol` enum assertion, end-to-end SSH banner test against the running binary
|
||||
|
||||
### Changed
|
||||
- **Scanner bucket walkers are no longer tied to the disk count**: a new scanner key `walkers_per_disk` (`RUSTFS_SCANNER_WALKERS_PER_DISK`, `1` through `16`, default `1`) spawns that many bucket walkers per local disk of an erasure set, so a single-disk deployment can walk several buckets at once instead of one bucket, one object at a time. The walkers of one disk share its scanner admission, so a peer still sees one scan session per physical disk, and remote disks keep one walker. The walker count is what `max_concurrent_disk_scans` caps and the disk count stays the floor; the `disk_concurrency_budget` log line now reports `walkers_per_disk`, `walkers`, and the `reason` for the effective limit. The default keeps today's I/O profile because a local disk cannot yet report whether it is rotational; the design intends a higher default on flash once it can (`docs/architecture/scanner-per-bucket-convergence.md`). As groundwork for pacing, the raw-enumeration page writer inserts into its building page in place and commits pages without copying them, and a regression test pins the cost of recording 20,000 entries in one prefix so the quadratic bookkeeping removed by [#8114](https://github.com/rustfs/rustfs/pull/8114) (rustfs#8113) cannot return. No persisted format, protocol or admin API changes beyond the additional config key, which older releases preserve and ignore.
|
||||
- **Encryption and KMS work merged since `1.0.0-rc.5`** (entries were missing from this section):
|
||||
- **Persisted KMS configuration secrets** are sealed field-by-field with `RUSTFS_KMS_CONFIG_SECRET`. **When the variable is unset the secrets are persisted in cleartext and the server only warns** (`persisted KMS configuration carries cleartext secrets`); it never refuses the write. Set it, identically, on every node, and re-save the configuration to seal an existing one.
|
||||
- **New v2 ciphertext frame format** with per-frame index binding and final-frame authentication. Its write switch `RUSTFS_ENCRYPTION_FRAME_V2` is **off by default**: v2 frames are unreadable by nodes without v2 read support, and encrypted ciphertext travels verbatim through transition, decommission and SSE-C replication passthrough, so turn it on only after every node — and every RustFS warm/replication target that receives raw ciphertext — runs a release with v2 read support. Reading v2 objects needs no switch.
|
||||
|
||||
Generated
+61
-60
@@ -30,7 +30,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99"
|
||||
dependencies = [
|
||||
"crypto-common 0.2.2",
|
||||
"inout 0.2.2",
|
||||
"inout 0.2.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -328,9 +328,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "archive-trait"
|
||||
version = "0.0.14"
|
||||
version = "0.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6080ea14ccf9019d7ce572c319e581c20a805c9bdc6dbfb9b988da003cbd1a3"
|
||||
checksum = "999bf88cc483664cab77700ca2fdfb0415a73412abe284ef5bc2319d00938e00"
|
||||
dependencies = [
|
||||
"cap-std",
|
||||
"thiserror 2.0.21",
|
||||
@@ -371,9 +371,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
|
||||
|
||||
[[package]]
|
||||
name = "arrow"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774"
|
||||
checksum = "4c25317ebddb8f2e6f006759336c95e5ded847a446f3524c27853bfe7d77a69b"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -392,9 +392,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-arith"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e"
|
||||
checksum = "2ab555c0ffe9cb817041c0cee80baa9663a22225d0a035693579771f2693cadc"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -406,9 +406,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-array"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584"
|
||||
checksum = "2dae1d91737421e92c6dd7c7f9d0f2d25198fd6fc539738b628f4dad10adf533"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-buffer",
|
||||
@@ -425,9 +425,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-buffer"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182"
|
||||
checksum = "1fe9cd0064f2d7cf8ecfaf77cd70e67a980bc71aea92b63102bd37a7d3379a09"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"half",
|
||||
@@ -437,9 +437,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-cast"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53"
|
||||
checksum = "ee510bc252781d700400979a96bd1eadebeae23901132bbabd92118da1b376c5"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -459,9 +459,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-csv"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86"
|
||||
checksum = "ace83d5935421b950fea3a0a3f9972c0837c297ef98ade57127a5661b681112c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-cast",
|
||||
@@ -474,9 +474,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-data"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358"
|
||||
checksum = "54f128c73bb7fe9f7137218f910edd5b436f390e0f040b1956f5028390787897"
|
||||
dependencies = [
|
||||
"arrow-buffer",
|
||||
"arrow-schema",
|
||||
@@ -487,9 +487,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ipc"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599"
|
||||
checksum = "8e244f39b82fa7a823e8b5984a0bd988234f9c92c1df79378fb85090e6e4c821"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -503,9 +503,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-json"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1"
|
||||
checksum = "727b21ebf8894dcd9504d6ab66bffdac338391e1f85cde594526a0210299ea40"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -528,9 +528,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-ord"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5"
|
||||
checksum = "70d376b9b883239ae0f9963cdeeb59126ccaac08d60adf208a6ad3e2dd4048c3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -541,9 +541,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-row"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2"
|
||||
checksum = "48096811f92b5c3a0c59d33cfecd7d7d2ed037d81e54a27a42d232e7700e6de2"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -554,9 +554,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-schema"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd"
|
||||
checksum = "37b6c8a6b12cf3869d299ce36ed8e1cd6fe9836264b68fb6c6538473a36124a1"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_json",
|
||||
@@ -564,9 +564,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-select"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8"
|
||||
checksum = "e523e85fa9fb3181a391bf0edfa6a1ae0dc46df8cbf7f6b96034dcb1e2a7a04a"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-array",
|
||||
@@ -578,9 +578,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "arrow-string"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14"
|
||||
checksum = "b931a905a0e4b8832d9f54fb369cf6e545f9dc03cb3d9611216a7bd2653df60a"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -2100,7 +2100,7 @@ checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c"
|
||||
dependencies = [
|
||||
"block-buffer 0.12.1",
|
||||
"crypto-common 0.2.2",
|
||||
"inout 0.2.2",
|
||||
"inout 0.2.3",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -2179,9 +2179,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.6.8"
|
||||
version = "4.6.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e"
|
||||
checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -3608,18 +3608,18 @@ checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2"
|
||||
|
||||
[[package]]
|
||||
name = "defmt"
|
||||
version = "0.3.100"
|
||||
version = "0.3.101"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0963443817029b2024136fc4dd07a5107eb8f977eaf18fcd1fdeb11306b64ad"
|
||||
checksum = "4deef977660e373c458df605a913d4ab121db0ccd18d1ad6976dff7d37cddb53"
|
||||
dependencies = [
|
||||
"defmt 1.1.1",
|
||||
"defmt 1.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "defmt"
|
||||
version = "1.1.1"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"
|
||||
checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"defmt-macros",
|
||||
@@ -3627,21 +3627,21 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "defmt-macros"
|
||||
version = "1.1.1"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"
|
||||
checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366"
|
||||
dependencies = [
|
||||
"defmt-parser",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "defmt-parser"
|
||||
version = "1.0.0"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
|
||||
checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
@@ -4371,7 +4371,7 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d8d873f78e1d3e1998ed2084da9147e3cd2bb34b67814cd49a3902b970eb124"
|
||||
dependencies = [
|
||||
"defmt 0.3.100",
|
||||
"defmt 0.3.101",
|
||||
"serde",
|
||||
]
|
||||
|
||||
@@ -5707,9 +5707,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "inout"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7"
|
||||
checksum = "d8c0d789294fb457b791f7797a04478704788f30b407e5d0b01deaa846572d9f"
|
||||
dependencies = [
|
||||
"block-padding 0.4.2",
|
||||
"hybrid-array",
|
||||
@@ -5852,7 +5852,7 @@ version = "0.2.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383"
|
||||
dependencies = [
|
||||
"defmt 1.1.1",
|
||||
"defmt 1.2.0",
|
||||
"jiff-core",
|
||||
"jiff-static",
|
||||
"jiff-tzdb-platform",
|
||||
@@ -5869,7 +5869,7 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c"
|
||||
dependencies = [
|
||||
"defmt 1.1.1",
|
||||
"defmt 1.2.0",
|
||||
"log",
|
||||
]
|
||||
|
||||
@@ -7681,9 +7681,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "parquet"
|
||||
version = "59.3.0"
|
||||
version = "59.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73"
|
||||
checksum = "8b3d2ef0fb136e9341570f219ecb168902e76079c6d81ad555f0a4840b1e08a3"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow-array",
|
||||
@@ -9380,7 +9380,7 @@ dependencies = [
|
||||
"ghash",
|
||||
"hex-literal",
|
||||
"hmac 0.13.0",
|
||||
"inout 0.2.2",
|
||||
"inout 0.2.3",
|
||||
"keccak",
|
||||
"log",
|
||||
"md5",
|
||||
@@ -9893,6 +9893,7 @@ dependencies = [
|
||||
"rustfs-rio",
|
||||
"rustfs-rio-v2",
|
||||
"rustfs-s3-client",
|
||||
"rustfs-s3-ops",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-scanner-metrics",
|
||||
"rustfs-storage-api",
|
||||
@@ -11180,9 +11181,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.23.45"
|
||||
version = "0.23.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
|
||||
checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
@@ -12294,9 +12295,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "suppaftp"
|
||||
version = "12.2.0"
|
||||
version = "12.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a560e9f42cc5bd4a92295eca3c7140d51eb685d688087f547f27e6426fbe97da"
|
||||
checksum = "2b58fcf921745cca6f0127c9f822069947da8c6dbeff52e9004750bcc448171f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"chrono",
|
||||
@@ -12470,9 +12471,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tar-codec"
|
||||
version = "0.0.14"
|
||||
version = "0.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ea42eb144d30fcbf32c26dfea8959175bb8335bfb7b18d20c6ed32edecf0551"
|
||||
checksum = "b9dad615e50d181afea4bafea23dacb8e33ed52dde06503813570634b932d467"
|
||||
dependencies = [
|
||||
"archive-trait",
|
||||
"tar-framing",
|
||||
@@ -12482,9 +12483,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tar-framing"
|
||||
version = "0.0.14"
|
||||
version = "0.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "783223a7a6590be4227cb821e7ce80372575511f67c824921aba0752d8ad5573"
|
||||
checksum = "563e8181b47c9cdcce569eae3ad431eec71543d6269dd60c7bd2350f70bcd233"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
"tokio",
|
||||
|
||||
+5
-5
@@ -155,7 +155,7 @@ futures-core = "0.3.34"
|
||||
futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.9.0" }
|
||||
pulsar = { default-features = false, version = "=6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.2" }
|
||||
hyper = { version = "1.12.0" }
|
||||
h2 = { version = "0.4.20" }
|
||||
@@ -220,7 +220,7 @@ openidconnect = { default-features = false, version = "4.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
rustls = { default-features = false, version = "0.23.45" }
|
||||
rustls = { default-features = false, version = "0.23.46" }
|
||||
rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.15.1"
|
||||
x509-parser = "0.18.1"
|
||||
@@ -246,8 +246,8 @@ anyhow = "1.0.104"
|
||||
arc-swap = "1.9.2"
|
||||
rustfs-tokio-tar = { version = "0.7.1" }
|
||||
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
|
||||
tar-codec = "0.0.14"
|
||||
tar-framing = "0.0.14"
|
||||
tar-codec = "0.0.15"
|
||||
tar-framing = "0.0.15"
|
||||
atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.12.0" }
|
||||
@@ -373,7 +373,7 @@ pyroscope = { version = "2.1.2" }
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.1" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.0" }
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.64.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
@@ -141,7 +141,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.2
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
@@ -220,7 +220,7 @@ For developers who want to build RustFS Docker images from source with multi-arc
|
||||
./docker-buildx.sh --push
|
||||
|
||||
# Build specific version
|
||||
./docker-buildx.sh --release 1.0.1 --push
|
||||
./docker-buildx.sh --release 1.0.2 --push
|
||||
|
||||
# Build for custom registry
|
||||
./docker-buildx.sh --registry your-registry.com --namespace yourname --push
|
||||
@@ -238,7 +238,7 @@ You can also use Make targets for convenience:
|
||||
```bash
|
||||
make docker-buildx # Build locally
|
||||
make docker-buildx-push # Build and push
|
||||
make docker-buildx-version VERSION=1.0.1 # Build specific version
|
||||
make docker-buildx-version VERSION=1.0.2 # Build specific version
|
||||
make help-docker # Show all Docker-related commands
|
||||
```
|
||||
|
||||
|
||||
+3
-3
@@ -138,7 +138,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.1
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.2
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
@@ -176,7 +176,7 @@ docker compose -f docker-compose-simple.yml up -d
|
||||
./docker-buildx.sh --push
|
||||
|
||||
# 构建指定版本
|
||||
./docker-buildx.sh --release 1.0.1 --push
|
||||
./docker-buildx.sh --release 1.0.2 --push
|
||||
|
||||
# 构建并推送到自定义仓库
|
||||
./docker-buildx.sh --registry your-registry.com --namespace yourname --push
|
||||
@@ -193,7 +193,7 @@ docker compose -f docker-compose-simple.yml up -d
|
||||
```bash
|
||||
make docker-buildx # 本地构建
|
||||
make docker-buildx-push # 构建并推送
|
||||
make docker-buildx-version VERSION=1.0.1 # 构建指定版本
|
||||
make docker-buildx-version VERSION=1.0.2 # 构建指定版本
|
||||
make help-docker # 显示所有 Docker 相关命令
|
||||
```
|
||||
|
||||
|
||||
@@ -119,11 +119,19 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_WALKERS_PER_DISK` (canonical)
|
||||
|
||||
Scanner cycle budget controls:
|
||||
|
||||
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
|
||||
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
|
||||
Scanner walker controls:
|
||||
|
||||
- `RUSTFS_SCANNER_WALKERS_PER_DISK` sets how many bucket walkers each local disk of an erasure set runs; the default `1` walks one bucket per disk at a time, which is the right value for rotational media. On flash, `2` to `4` lets one disk walk several buckets concurrently, which matters most for single-disk deployments with many buckets, where the walk is otherwise fully serial. Values outside `1` through `16` are rejected by `admin config set`; an out-of-range or non-numeric environment value is logged as a scanner runtime config error (`state = "startup_apply_failed"`) and the scanner keeps the default. The walkers of one disk share that disk's scanner admission, so peers still see one scan session per physical disk, and `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` still caps the total.
|
||||
- The same value is available as the persistent `scanner.walkers_per_disk` key and is reported under `runtime_config.walkers_per_disk` in the admin scanner status; see [Scanner Runtime Controls](../../docs/operations/scanner-runtime-controls.md).
|
||||
|
||||
- A timed-out cycle cancels cooperative scanner work, then fences its leader epoch before releasing the lease. An uncooperative I/O operation is dropped after the bounded shutdown window; its cursor is not claimed to be durable and the scanner reports `recovery-required` when the worker cannot stop cooperatively, the cycle state was not confirmed durable, or epoch fencing cannot be persisted.
|
||||
|
||||
## Mmap read environment aliases
|
||||
|
||||
@@ -59,6 +59,9 @@ pub const SCANNER_MAX_CONCURRENT_SET_SCANS: &str = "max_concurrent_set_scans";
|
||||
/// Scanner config key capping concurrent scanner disk bucket walks per set.
|
||||
pub const SCANNER_MAX_CONCURRENT_DISK_SCANS: &str = "max_concurrent_disk_scans";
|
||||
|
||||
/// Scanner config key setting how many bucket walkers each local disk runs.
|
||||
pub const SCANNER_WALKERS_PER_DISK: &str = "walkers_per_disk";
|
||||
|
||||
/// Scanner config key controlling how often object loops yield.
|
||||
pub const SCANNER_YIELD_EVERY_N_OBJECTS: &str = "yield_every_n_objects";
|
||||
|
||||
@@ -86,6 +89,7 @@ pub const SCANNER_KEYS: &[&str] = &[
|
||||
SCANNER_CACHE_SAVE_TIMEOUT,
|
||||
SCANNER_MAX_CONCURRENT_SET_SCANS,
|
||||
SCANNER_MAX_CONCURRENT_DISK_SCANS,
|
||||
SCANNER_WALKERS_PER_DISK,
|
||||
SCANNER_YIELD_EVERY_N_OBJECTS,
|
||||
SCANNER_ALERT_EXCESS_VERSIONS,
|
||||
SCANNER_ALERT_EXCESS_VERSION_SIZE,
|
||||
@@ -214,6 +218,14 @@ pub const ENV_SCANNER_MAX_CONCURRENT_SET_SCANS: &str = "RUSTFS_SCANNER_MAX_CONCU
|
||||
/// - Example: `export RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS=1`
|
||||
pub const ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS: &str = "RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS";
|
||||
|
||||
/// Environment variable that sets how many bucket walkers each local disk runs.
|
||||
/// The walkers of one disk share its scanner admission, so a peer still sees one
|
||||
/// scan session per physical disk. Remote disks keep one walker. The product of
|
||||
/// local disks and this factor is what `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS`
|
||||
/// caps; the disk count stays the floor. Valid range is `1` through `16`.
|
||||
/// - Example: `export RUSTFS_SCANNER_WALKERS_PER_DISK=4`
|
||||
pub const ENV_SCANNER_WALKERS_PER_DISK: &str = "RUSTFS_SCANNER_WALKERS_PER_DISK";
|
||||
|
||||
/// Environment variable that controls how often scanner object loops yield to the async runtime.
|
||||
/// A value of `0` disables this extra object-count yield.
|
||||
/// - Example: `export RUSTFS_SCANNER_YIELD_EVERY_N_OBJECTS=32`
|
||||
@@ -228,6 +240,18 @@ pub const DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS: usize = 4;
|
||||
/// Default disk scan concurrency budget.
|
||||
pub const DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS: usize = 4;
|
||||
|
||||
/// Default bucket walkers per local disk.
|
||||
///
|
||||
/// One walker per disk keeps today's I/O profile, which is what rotational
|
||||
/// media needs. Flash is meant to default higher once a local disk can report
|
||||
/// its media type; `rustfs_utils::os::Info::rotational` is never populated
|
||||
/// today, so every deployment starts from one walker.
|
||||
pub const DEFAULT_SCANNER_WALKERS_PER_DISK: usize = 1;
|
||||
|
||||
/// Upper bound for `walkers_per_disk`; a larger factor only multiplies
|
||||
/// contention on one spindle or NVMe queue without more disks to spread it.
|
||||
pub const MAX_SCANNER_WALKERS_PER_DISK: usize = 16;
|
||||
|
||||
/// Default object interval for cooperative scanner yields.
|
||||
pub const DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS: u64 = 128;
|
||||
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! S3 wire semantics for per-object conditional bulk deletion.
|
||||
|
||||
use crate::common::RustFSTestEnvironment;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::delete_objects::DeleteObjectsOutput;
|
||||
use aws_sdk_s3::operation::put_object::PutObjectOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
|
||||
use std::collections::BTreeSet;
|
||||
use std::error::Error;
|
||||
|
||||
type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
type VersionSnapshot = BTreeSet<(String, bool)>;
|
||||
|
||||
async fn put(client: &Client, bucket: &str, key: &str, body: &'static [u8]) -> TestResult<PutObjectOutput> {
|
||||
let result = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(body))
|
||||
.send()
|
||||
.await?;
|
||||
assert!(result.e_tag().is_some(), "fixture PUT must provide an ETag for the following conditions");
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn identifier(key: &str, etag: Option<&str>, version: Option<&str>) -> TestResult<ObjectIdentifier> {
|
||||
Ok(ObjectIdentifier::builder()
|
||||
.key(key)
|
||||
.set_e_tag(etag.map(str::to_owned))
|
||||
.set_version_id(version.map(str::to_owned))
|
||||
.build()?)
|
||||
}
|
||||
|
||||
async fn delete_batch(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
objects: Vec<ObjectIdentifier>,
|
||||
quiet: bool,
|
||||
) -> TestResult<DeleteObjectsOutput> {
|
||||
Ok(client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(Delete::builder().set_objects(Some(objects)).quiet(quiet).build()?)
|
||||
.send()
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn assert_body(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let result = client.get_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(result.body.collect().await?.into_bytes().as_ref(), expected);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_absent(client: &Client, bucket: &str, key: &str) {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted key must be absent");
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.code()), Some("NotFound"));
|
||||
}
|
||||
|
||||
async fn versions(client: &Client, bucket: &str, key: &str) -> TestResult<VersionSnapshot> {
|
||||
let result = client.list_object_versions().bucket(bucket).prefix(key).send().await?;
|
||||
assert!(!result.is_truncated().unwrap_or_default(), "focused version fixture fits a single page");
|
||||
let mut snapshot = BTreeSet::new();
|
||||
for version in result.versions().iter().filter(|version| version.key() == Some(key)) {
|
||||
snapshot.insert((version.version_id().expect("version has an identity").to_owned(), false));
|
||||
}
|
||||
for marker in result.delete_markers().iter().filter(|marker| marker.key() == Some(key)) {
|
||||
snapshot.insert((marker.version_id().expect("delete marker has an identity").to_owned(), true));
|
||||
}
|
||||
Ok(snapshot)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sdk_conditional_delete_objects_mixed_results_overwrites_duplicates_and_quiet() -> TestResult {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "conditional-bulk-mixed";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let old = put(&client, bucket, "overwritten", b"generation A").await?;
|
||||
let current = put(&client, bucket, "overwritten", b"generation B").await?;
|
||||
let matched = put(&client, bucket, "matched", b"matching object").await?;
|
||||
put(&client, bucket, "exists", b"existence object").await?;
|
||||
put(&client, bucket, "unconditional", b"unconditional object").await?;
|
||||
put(&client, bucket, "empty", b"empty condition must preserve this").await?;
|
||||
put(&client, bucket, "empty-quoted", b"quoted empty condition must preserve this").await?;
|
||||
let weak = put(&client, bucket, "weak", b"weak condition must preserve this").await?;
|
||||
let weak_etag = format!("W/\"{}\"", weak.e_tag().expect("PUT supplies an ETag").trim_matches('"'));
|
||||
assert_ne!(old.e_tag(), current.e_tag(), "overwrite fixture must have distinct ETags");
|
||||
let matched_etag = matched.e_tag().expect("PUT supplies an ETag");
|
||||
let result = delete_batch(
|
||||
&client,
|
||||
bucket,
|
||||
vec![
|
||||
identifier("matched", Some(matched_etag), None)?,
|
||||
identifier("overwritten", old.e_tag(), None)?,
|
||||
identifier("matched", Some("stale-etag"), None)?,
|
||||
identifier("matched", Some(matched_etag), None)?,
|
||||
identifier("unconditional", None, None)?,
|
||||
identifier("exists", Some("*"), None)?,
|
||||
identifier("missing", Some("*"), None)?,
|
||||
identifier("empty", Some(""), None)?,
|
||||
identifier("empty-quoted", Some("\"\""), None)?,
|
||||
identifier("weak", Some(&weak_etag), None)?,
|
||||
identifier("unconditional-missing", None, None)?,
|
||||
identifier("missing-specific", Some("opaque-etag"), None)?,
|
||||
],
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
result.deleted().iter().map(|object| object.key()).collect::<Vec<_>>(),
|
||||
[
|
||||
Some("matched"),
|
||||
Some("matched"),
|
||||
Some("unconditional"),
|
||||
Some("exists"),
|
||||
Some("unconditional-missing")
|
||||
],
|
||||
"duplicate successful inputs retain their separate response entries"
|
||||
);
|
||||
assert_eq!(
|
||||
result
|
||||
.errors()
|
||||
.iter()
|
||||
.map(|error| (error.key(), error.code()))
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
(Some("overwritten"), Some("PreconditionFailed")),
|
||||
(Some("matched"), Some("PreconditionFailed")),
|
||||
(Some("missing"), Some("NoSuchKey")),
|
||||
(Some("empty"), Some("InvalidArgument")),
|
||||
(Some("empty-quoted"), Some("InvalidArgument")),
|
||||
(Some("weak"), Some("InvalidArgument")),
|
||||
(Some("missing-specific"), Some("NoSuchKey")),
|
||||
]
|
||||
);
|
||||
assert_body(&client, bucket, "overwritten", b"generation B").await?;
|
||||
assert_body(&client, bucket, "empty", b"empty condition must preserve this").await?;
|
||||
assert_body(&client, bucket, "empty-quoted", b"quoted empty condition must preserve this").await?;
|
||||
assert_body(&client, bucket, "weak", b"weak condition must preserve this").await?;
|
||||
for key in ["matched", "unconditional", "exists"] {
|
||||
assert_absent(&client, bucket, key).await;
|
||||
}
|
||||
|
||||
let quiet = put(&client, bucket, "quiet-match", b"quiet matching object").await?;
|
||||
put(&client, bucket, "quiet-rejected", b"quiet rejected object").await?;
|
||||
let result = delete_batch(
|
||||
&client,
|
||||
bucket,
|
||||
vec![
|
||||
identifier("quiet-match", quiet.e_tag(), None)?,
|
||||
identifier("quiet-match", quiet.e_tag(), None)?,
|
||||
identifier("quiet-rejected", Some("stale-etag"), None)?,
|
||||
identifier("missing", Some("*"), None)?,
|
||||
],
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
assert!(result.deleted().is_empty(), "Quiet suppresses successful entries");
|
||||
assert_eq!(
|
||||
result
|
||||
.errors()
|
||||
.iter()
|
||||
.map(|error| (error.key(), error.code()))
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
(Some("quiet-rejected"), Some("PreconditionFailed")),
|
||||
(Some("missing"), Some("NoSuchKey"))
|
||||
]
|
||||
);
|
||||
assert_absent(&client, bucket, "quiet-match").await;
|
||||
assert_body(&client, bucket, "quiet-rejected", b"quiet rejected object").await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sdk_conditional_delete_objects_versioning_current_markers_and_explicit_versions() -> TestResult {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
let client = env.create_s3_client();
|
||||
for suspended in [false, true] {
|
||||
let bucket = if suspended {
|
||||
"conditional-bulk-suspended"
|
||||
} else {
|
||||
"conditional-bulk-enabled"
|
||||
};
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let historical = put(&client, bucket, "history", b"historical payload").await?;
|
||||
let invalid_historical = put(&client, bucket, "invalid", b"historical rejected payload").await?;
|
||||
if suspended {
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Suspended)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
let current = put(&client, bucket, "history", b"current payload").await?;
|
||||
let invalid = put(&client, bucket, "invalid", b"current rejected payload").await?;
|
||||
let invalid_before = versions(&client, bucket, "invalid").await?;
|
||||
let historical_id = historical.version_id().expect("enabled fixture has a UUID version");
|
||||
let invalid_historical_id = invalid_historical
|
||||
.version_id()
|
||||
.expect("enabled rejected fixture has a UUID version");
|
||||
let result = delete_batch(
|
||||
&client,
|
||||
bucket,
|
||||
vec![
|
||||
identifier("history", current.e_tag(), None)?,
|
||||
identifier("history", None, Some(historical_id))?,
|
||||
identifier("history", current.e_tag(), None)?,
|
||||
identifier("invalid", invalid.e_tag(), Some(invalid_historical_id))?,
|
||||
identifier("invalid", Some("*"), Some("null"))?,
|
||||
identifier("invalid", Some("stale-etag"), None)?,
|
||||
],
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(result.deleted().len(), 3, "both current input entries and the historical delete succeed");
|
||||
assert_eq!(
|
||||
result
|
||||
.errors()
|
||||
.iter()
|
||||
.map(|error| (error.key(), error.code()))
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
(Some("invalid"), Some("InvalidRequest")),
|
||||
(Some("invalid"), Some("InvalidRequest")),
|
||||
(Some("invalid"), Some("PreconditionFailed")),
|
||||
]
|
||||
);
|
||||
assert_eq!(result.errors()[0].version_id(), Some(invalid_historical_id));
|
||||
assert_eq!(result.errors()[1].version_id(), Some("null"));
|
||||
assert_eq!(versions(&client, bucket, "invalid").await?, invalid_before);
|
||||
assert_body(&client, bucket, "invalid", b"current rejected payload").await?;
|
||||
|
||||
let marker = result.deleted()[0]
|
||||
.delete_marker_version_id()
|
||||
.expect("current conditional delete returns a marker");
|
||||
assert_eq!(result.deleted()[0].delete_marker(), Some(true));
|
||||
assert_eq!(result.deleted()[2].delete_marker_version_id(), Some(marker));
|
||||
assert_eq!(result.deleted()[1].version_id(), Some(historical_id));
|
||||
if suspended {
|
||||
assert_eq!(marker, "null", "a suspended current delete returns the created null marker identity");
|
||||
}
|
||||
let history_after = versions(&client, bucket, "history").await?;
|
||||
assert_eq!(
|
||||
history_after.iter().filter(|(_, marker)| *marker).count(),
|
||||
1,
|
||||
"duplicate actions create one marker"
|
||||
);
|
||||
assert!(
|
||||
!history_after.contains(&(historical_id.to_string(), false)),
|
||||
"distinct historical target is removed"
|
||||
);
|
||||
if suspended {
|
||||
assert_eq!(history_after, BTreeSet::from([("null".to_string(), true)]));
|
||||
} else {
|
||||
assert_eq!(
|
||||
history_after,
|
||||
BTreeSet::from([
|
||||
(current.version_id().expect("current enabled object has a UUID").to_string(), false),
|
||||
(marker.to_string(), true),
|
||||
]),
|
||||
"enabled conditional current deletion retains the current data version behind the marker"
|
||||
);
|
||||
}
|
||||
|
||||
let result = delete_batch(
|
||||
&client,
|
||||
bucket,
|
||||
vec![
|
||||
identifier("history", Some("*"), None)?,
|
||||
identifier("history", current.e_tag(), None)?,
|
||||
identifier("missing", Some("*"), None)?,
|
||||
],
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
assert!(
|
||||
result.deleted().is_empty(),
|
||||
"current markers and absent objects cannot satisfy a live-object condition"
|
||||
);
|
||||
assert_eq!(result.errors().len(), 3);
|
||||
assert!(result.errors().iter().all(|error| error.code() == Some("NoSuchKey")));
|
||||
assert_eq!(
|
||||
versions(&client, bucket, "history").await?,
|
||||
history_after,
|
||||
"failed conditions create no marker"
|
||||
);
|
||||
assert!(versions(&client, bucket, "missing").await?.is_empty());
|
||||
|
||||
put(&client, bucket, "exists", b"versioned existence payload").await?;
|
||||
let result = delete_batch(&client, bucket, vec![identifier("exists", Some("*"), None)?], false).await?;
|
||||
assert!(result.errors().is_empty());
|
||||
assert_eq!(result.deleted().len(), 1);
|
||||
assert_eq!(result.deleted()[0].delete_marker(), Some(true));
|
||||
assert_eq!(
|
||||
versions(&client, bucket, "exists")
|
||||
.await?
|
||||
.iter()
|
||||
.filter(|(_, marker)| *marker)
|
||||
.count(),
|
||||
1
|
||||
);
|
||||
|
||||
if !suspended {
|
||||
let directory = put(&client, bucket, "dir/", b"directory null-version payload").await?;
|
||||
assert_eq!(
|
||||
versions(&client, bucket, "dir/").await?,
|
||||
BTreeSet::from([("null".to_string(), false)]),
|
||||
"the directory PUT fixture must identify the synthetic null data version"
|
||||
);
|
||||
let result = delete_batch(
|
||||
&client,
|
||||
bucket,
|
||||
vec![identifier("dir/", directory.e_tag(), None)?, identifier("dir/", None, None)?],
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
assert!(result.errors().is_empty(), "both distinct directory actions must succeed");
|
||||
assert_eq!(result.deleted().len(), 2);
|
||||
assert_eq!(result.deleted()[0].key(), Some("dir/"));
|
||||
assert_eq!(result.deleted()[1].key(), Some("dir/"));
|
||||
assert_eq!(result.deleted()[0].delete_marker(), Some(true));
|
||||
let marker = result.deleted()[0]
|
||||
.delete_marker_version_id()
|
||||
.expect("conditional current-directory deletion creates a versioned marker");
|
||||
assert!(
|
||||
!uuid::Uuid::parse_str(marker)?.is_nil(),
|
||||
"enabled current-directory deletion creates a UUID marker"
|
||||
);
|
||||
assert_eq!(result.deleted()[1].delete_marker(), None);
|
||||
assert_eq!(result.deleted()[1].delete_marker_version_id(), None);
|
||||
assert_eq!(result.deleted()[1].version_id(), None);
|
||||
assert_eq!(
|
||||
versions(&client, bucket, "dir/").await?,
|
||||
BTreeSet::from([(marker.to_string(), true)]),
|
||||
"current-marker creation and synthetic null-version removal are distinct actions: one marker remains and the null data version is gone"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -860,10 +860,12 @@ pub(crate) async fn start_rebalance(cluster: &RustFSTestClusterEnvironment) -> T
|
||||
if status.is_success() {
|
||||
break response;
|
||||
}
|
||||
if status == StatusCode::INTERNAL_SERVER_ERROR
|
||||
&& response.contains("requires a live fleet capability proof")
|
||||
&& Instant::now() < deadline
|
||||
{
|
||||
// The init stage answers 503 + Retry-After while the cluster capability
|
||||
// proof is still missing; older binaries surfaced the same condition as
|
||||
// a 500 carrying the proof marker.
|
||||
let waiting_for_readiness = status == StatusCode::SERVICE_UNAVAILABLE
|
||||
|| (status == StatusCode::INTERNAL_SERVER_ERROR && response.contains("requires a live fleet capability proof"));
|
||||
if waiting_for_readiness && Instant::now() < deadline {
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ This module is the shared failure-injection boundary for replication end-to-end
|
||||
|
||||
Supported data operations are HeadBucket, GetBucketVersioning, ListObjectsV2, PUT/GET/HEAD/DELETE Object, Get/Put/Delete ObjectTagging (tags live per version; Put replaces the whole set, Delete clears it), and create/upload/complete/abort multipart upload. `create_bucket` models general-purpose buckets in S3's shared global namespace; account-regional namespace buckets and their `-an` names are intentionally out of scope. Buckets created with `create_bucket` are versioned: PUT creates a version, DELETE without `versionId` creates a delete marker, and DELETE with `versionId` removes exactly that version. Internal source version IDs must be UUIDs and are stored canonically. Source mtime is honored only for source-replication PUT/DELETE requests; absent or invalid values use receipt time, matching RustFS, while multipart completion always uses receipt time. Replicated versions are ordered newest-first by source mtime so late older versions and delete markers do not become current. Equal mtimes prefer objects over delete markers, then canonical UUID order; RustFS's internal FileMeta signature tie-break is intentionally out of scope because it is not part of the target S3 protocol. Multipart part numbers follow S3's `1..=10000` range, and every completed part except the final part must be at least 5 MiB.
|
||||
|
||||
`create_bucket_with_object_lock(name)` creates a versioned bucket whose GetObjectLockConfiguration reports `Enabled`; every other bucket answers `ObjectLockConfigurationNotFoundError`, the code RustFS's replication-check classifies as "not enabled". Three switches model remote-target behaviors the fleet has shown, so the outbound target matrix (`crates/e2e_test/src/replication_target_matrix_test.rs`) can replicate every object shape against each: `assign_own_version_ids(true)` ignores the source version id and mints its own (AWS S3 / Wasabi); `reject_aws_chunked_uploads(true)` refuses any PutObject or UploadPart announcing `aws-chunked` framing (`Content-Encoding: aws-chunked`, an `x-amz-trailer`, or a `STREAMING-*` payload hash) with `InvalidRequest` before the body is read (SeaweedFS 3.97, rustfs#6853); `require_checksum_for_object_lock(true)` rejects a PutObject carrying any `x-amz-object-lock-*` header unless it also carries `Content-MD5`, an `x-amz-checksum-*` header, or `x-amz-sdk-checksum-algorithm` (AWS S3 / MinIO, rustfs#7082). Independently of that switch, a `Content-MD5` header is always verified against the body and a mismatch answers `BadDigest`.
|
||||
`create_bucket_with_object_lock(name)` creates a versioned bucket whose GetObjectLockConfiguration reports `Enabled`; every other bucket answers `ObjectLockConfigurationNotFoundError`, the code RustFS's replication-check classifies as "not enabled". Four switches model remote-target behaviors the fleet has shown, so the outbound target matrix (`crates/e2e_test/src/replication_target_matrix_test.rs`) can replicate every object shape against each: `assign_own_version_ids(true)` ignores the source version id and mints its own (AWS S3 / Wasabi); `reject_aws_chunked_uploads(true)` refuses any PutObject or UploadPart announcing `aws-chunked` framing (`Content-Encoding: aws-chunked`, an `x-amz-trailer`, or a `STREAMING-*` payload hash) with `InvalidRequest` before the body is read (SeaweedFS 3.97, rustfs#6853); `require_checksum_for_object_lock(true)` rejects a PutObject carrying any `x-amz-object-lock-*` header unless it also carries `Content-MD5`, an `x-amz-checksum-*` header, or `x-amz-sdk-checksum-algorithm` (AWS S3 / MinIO, rustfs#7082). `ignore_source_etags(true)` ignores the `x-rustfs-source-etag` / `x-minio-source-etag` replication headers and reports the ETag of the received bytes (AWS S3 and other generic S3 services; the default adopts the source ETag like RustFS/MinIO). Independently of that switch, a `Content-MD5` header is always verified against the body and a mismatch answers `BadDigest`.
|
||||
|
||||
`create_bucket_with_mode(name, BucketMode::Unversioned)` models a plain migration source: PUT overwrites in place, DELETE removes the key without a delete marker, GetBucketVersioning reports no status, and no `x-amz-version-id` is returned by PUT, GET, HEAD, tagging, or multipart completion. The only `versionId` such a bucket accepts is `null`; any other value is rejected with `InvalidArgument`. The mode is fixed at creation.
|
||||
|
||||
|
||||
@@ -537,6 +537,10 @@ struct StoreState {
|
||||
/// PutObject carrying any `x-amz-object-lock-*` header must also carry
|
||||
/// `Content-MD5` or an `x-amz-checksum-*` header.
|
||||
require_checksum_for_object_lock: bool,
|
||||
/// Models AWS S3 and other generic S3 services: the `x-rustfs-source-etag`
|
||||
/// / `x-minio-source-etag` replication headers are ignored and every
|
||||
/// stored object reports the ETag computed from the bytes it received.
|
||||
ignore_source_etags: bool,
|
||||
/// Models Wasabi (rustfs/backlog#2340): a version-addressed DELETE of a
|
||||
/// version id the target never had answers 404 `NoSuchVersion` instead of
|
||||
/// the idempotent 204 RustFS/MinIO give.
|
||||
@@ -1052,6 +1056,13 @@ impl FakeS3Target {
|
||||
lock(&self.backend.store).require_checksum_for_object_lock = enabled;
|
||||
}
|
||||
|
||||
/// AWS-like mode: ignore the replication source-ETag headers and report
|
||||
/// the ETag of the received bytes, as AWS S3 does for SSE-S3 and
|
||||
/// unencrypted objects. The default adopts the source ETag (RustFS/MinIO).
|
||||
pub fn ignore_source_etags(&self, enabled: bool) {
|
||||
lock(&self.backend.store).ignore_source_etags = enabled;
|
||||
}
|
||||
|
||||
pub fn assign_own_multipart_version_ids(&self, enabled: bool) {
|
||||
lock(&self.backend.store).assign_own_multipart_version_ids = enabled;
|
||||
}
|
||||
@@ -2444,12 +2455,13 @@ impl S3 for FakeBackend {
|
||||
.map_err(|_| s3s::s3_error!(ServiceUnavailable, "fake target body limiter closed"))?;
|
||||
let headers = req.headers;
|
||||
let input = req.input;
|
||||
let (assign_own, drop_unlisted, require_checksum, limits, versioned) = {
|
||||
let (assign_own, drop_unlisted, require_checksum, ignore_source_etags, limits, versioned) = {
|
||||
let state = lock(&self.store);
|
||||
(
|
||||
state.assign_own_version_ids,
|
||||
state.drop_unlisted_replication_headers,
|
||||
state.require_checksum_for_object_lock,
|
||||
state.ignore_source_etags,
|
||||
state.limits,
|
||||
bucket_versioned(&state, &input.bucket),
|
||||
)
|
||||
@@ -2489,7 +2501,7 @@ impl S3 for FakeBackend {
|
||||
} else {
|
||||
NULL_VERSION_ID.to_string()
|
||||
};
|
||||
let e_tag = match source_etag(&headers)? {
|
||||
let e_tag = match source_etag(&headers)?.filter(|_| !ignore_source_etags) {
|
||||
Some(value) => value,
|
||||
None => {
|
||||
let (digest, permit) = md5_digest(body.clone(), body_permit).await?;
|
||||
@@ -3112,7 +3124,10 @@ impl S3 for FakeBackend {
|
||||
.collect();
|
||||
let (body, digests, _body_permits) = assemble_multipart(assembly_parts, total_len, _body_permits).await?;
|
||||
let part_count = requested.len();
|
||||
let e_tag = source_etag(&headers)?.unwrap_or_else(|| format!("{}-{part_count}", md5_hex(digests)));
|
||||
let ignore_source_etags = lock(&self.store).ignore_source_etags;
|
||||
let e_tag = source_etag(&headers)?
|
||||
.filter(|_| !ignore_source_etags)
|
||||
.unwrap_or_else(|| format!("{}-{part_count}", md5_hex(digests)));
|
||||
let version = ObjectVersion {
|
||||
version_id: upload.version_id.clone(),
|
||||
body,
|
||||
|
||||
@@ -1032,7 +1032,7 @@ mod tests {
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/{}?forceStart=true", env.url, bucket);
|
||||
signed_admin_post(&heal_url, Some(heal_body), &env.access_key, &env.secret_key)
|
||||
start_root_heal_when_control_ready(&heal_url, heal_body, &env.access_key, &env.secret_key)
|
||||
.await
|
||||
.expect("admin deep heal should be accepted");
|
||||
|
||||
|
||||
@@ -179,6 +179,9 @@ mod compression_test;
|
||||
#[cfg(test)]
|
||||
mod delete_objects_versioning_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod conditional_delete_objects_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod delete_authorization_test;
|
||||
|
||||
|
||||
@@ -851,13 +851,20 @@ mod integration_tests {
|
||||
.delete(
|
||||
aws_sdk_s3::types::Delete::builder()
|
||||
.set_objects(Some(objects))
|
||||
.quiet(true)
|
||||
// Quiet mode hides successful entries, so the batch must be
|
||||
// verbose for the assertion on the deleted list below.
|
||||
.quiet(false)
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert!(
|
||||
delete_result.errors().is_empty(),
|
||||
"batch delete must not report errors: {:?}",
|
||||
delete_result.errors()
|
||||
);
|
||||
assert_eq!(delete_result.deleted().len(), 2);
|
||||
|
||||
// Now should be able to upload again (quota freed up)
|
||||
|
||||
@@ -113,6 +113,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn acquire_quota_mutation_fence(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn renew_snapshot_lease(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseRenewRequest>,
|
||||
@@ -127,6 +134,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn release_quota_mutation_fence(
|
||||
&self,
|
||||
_request: Request<SnapshotLeaseReleaseRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("MinimalLockNodeService only supports lock RPCs"))
|
||||
}
|
||||
|
||||
async fn acquire_scanner_publication_lease(
|
||||
&self,
|
||||
_request: Request<ScannerPublicationLeaseRequest>,
|
||||
@@ -541,6 +555,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data_with_quota_fence(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn make_volumes(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MakeVolumesRequest>,
|
||||
@@ -639,6 +660,13 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn guarded_delete_versions(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::GuardedDeleteVersionsRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::DeleteVersionsResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn delete_versions(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::DeleteVersionsRequest>,
|
||||
|
||||
@@ -45,9 +45,10 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::{ByteStream, DateTime};
|
||||
use aws_sdk_s3::types::{
|
||||
Checksum, ChecksumAlgorithm, CompletedMultipartUpload, CompletedPart, ObjectAttributes, ObjectLockLegalHold,
|
||||
ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, Tag, Tagging,
|
||||
ObjectLockLegalHoldStatus, ObjectLockMode, ObjectLockRetention, ObjectLockRetentionMode, ServerSideEncryption, Tag, Tagging,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use md5::{Digest as _, Md5};
|
||||
use std::error::Error;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
@@ -73,14 +74,20 @@ enum TargetMode {
|
||||
/// land, and every version-addressed mutation must resolve the replica
|
||||
/// through the target-version ledger.
|
||||
MintOwnVersionIds,
|
||||
/// AWS S3 and other generic S3 services: ignores the replication
|
||||
/// source-ETag headers and reports the MD5 of the plaintext it received.
|
||||
/// Every single-part replica ETag is therefore checked against the source
|
||||
/// ETag for real (the other modes echo the source ETag back).
|
||||
ComputesOwnEtags,
|
||||
}
|
||||
|
||||
impl TargetMode {
|
||||
const ALL: [TargetMode; 4] = [
|
||||
const ALL: [TargetMode; 5] = [
|
||||
TargetMode::Baseline,
|
||||
TargetMode::RejectAwsChunked,
|
||||
TargetMode::RequireChecksumWithObjectLock,
|
||||
TargetMode::MintOwnVersionIds,
|
||||
TargetMode::ComputesOwnEtags,
|
||||
];
|
||||
|
||||
fn apply(self, target: &FakeS3Target) {
|
||||
@@ -92,6 +99,7 @@ impl TargetMode {
|
||||
target.assign_own_version_ids(true);
|
||||
target.reject_unknown_version_deletes(true);
|
||||
}
|
||||
TargetMode::ComputesOwnEtags => target.ignore_source_etags(true),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,6 +109,7 @@ impl TargetMode {
|
||||
TargetMode::RejectAwsChunked => "reject-aws-chunked",
|
||||
TargetMode::RequireChecksumWithObjectLock => "require-checksum-object-lock",
|
||||
TargetMode::MintOwnVersionIds => "mint-own-version-ids",
|
||||
TargetMode::ComputesOwnEtags => "computes-own-etags",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -126,10 +135,14 @@ enum ObjectShape {
|
||||
/// Single-part object uploaded with `x-amz-checksum-sha256`; the replica
|
||||
/// must carry the same header (rustfs/backlog#2340).
|
||||
Checksummed,
|
||||
/// Single-part SSE-S3 object. Its ETag is a keyed fingerprint, not the
|
||||
/// plaintext MD5 (docs/operations/encrypted-etags.md), while the replica
|
||||
/// travels as plaintext under `x-amz-server-side-encryption: AES256`.
|
||||
SseS3,
|
||||
}
|
||||
|
||||
impl ObjectShape {
|
||||
const ALL: [ObjectShape; 8] = [
|
||||
const ALL: [ObjectShape; 9] = [
|
||||
ObjectShape::Empty,
|
||||
ObjectShape::Plain,
|
||||
ObjectShape::Retention,
|
||||
@@ -138,6 +151,7 @@ impl ObjectShape {
|
||||
ObjectShape::LockedMultipart,
|
||||
ObjectShape::OdmPreservedMd5Multipart,
|
||||
ObjectShape::Checksummed,
|
||||
ObjectShape::SseS3,
|
||||
];
|
||||
|
||||
fn key(self) -> &'static str {
|
||||
@@ -150,6 +164,7 @@ impl ObjectShape {
|
||||
ObjectShape::LockedMultipart => "matrix/locked-multipart.bin",
|
||||
ObjectShape::OdmPreservedMd5Multipart => "matrix/odm-preserved-md5.bin",
|
||||
ObjectShape::Checksummed => "matrix/checksummed.bin",
|
||||
ObjectShape::SseS3 => "matrix/sse-s3.bin",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,6 +248,24 @@ impl ObjectShape {
|
||||
.await?;
|
||||
Ok(body)
|
||||
}
|
||||
ObjectShape::SseS3 => {
|
||||
let body = payload(56 * 1024, 0x77);
|
||||
let put = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await?;
|
||||
// The cell only proves something while the source ETag is
|
||||
// not the content MD5 a generic target reports back.
|
||||
let plaintext_md5 = hex_simd::encode_to_string(Md5::digest(&body), hex_simd::AsciiCase::Lower);
|
||||
if put.e_tag().map(|etag| etag.trim_matches('"')) == Some(plaintext_md5.as_str()) {
|
||||
return Err("SSE-S3 source ETag is the plaintext MD5; the cell no longer exercises encrypted ETags".into());
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -270,14 +303,40 @@ fn expectation(mode: TargetMode, shape: ObjectShape) -> Expectation {
|
||||
/// - the first PUT lands but its response is lost, so the object is FAILED
|
||||
/// and the scanner heal pass re-drives it;
|
||||
/// - an existing-object resync re-drives a COMPLETED object unconditionally.
|
||||
///
|
||||
/// A permanent delete of the healed version then has to remove exactly the
|
||||
/// replica the heal pass located.
|
||||
#[tokio::test]
|
||||
async fn matrix_mint_own_version_ids_redrive_does_not_duplicate() -> TestResult {
|
||||
run_redrive_does_not_duplicate(RedriveObjects::Plaintext).await
|
||||
}
|
||||
|
||||
/// The same re-drives for SSE-S3 objects against a target that also computes
|
||||
/// its own ETags (AWS S3). The source ETag is a keyed fingerprint
|
||||
/// (docs/operations/encrypted-etags.md) while the replica reports the MD5 of
|
||||
/// the plaintext it received, so no exact-ETag lookup can ever find the
|
||||
/// replica: the lost-response heal, the resync and the purge must locate it
|
||||
/// without comparing ETags.
|
||||
#[tokio::test]
|
||||
async fn matrix_mint_own_version_ids_encrypted_redrive_does_not_duplicate() -> TestResult {
|
||||
run_redrive_does_not_duplicate(RedriveObjects::SseS3OnEtagComputingTarget).await
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum RedriveObjects {
|
||||
Plaintext,
|
||||
SseS3OnEtagComputingTarget,
|
||||
}
|
||||
|
||||
async fn run_redrive_does_not_duplicate(objects: RedriveObjects) -> TestResult {
|
||||
init_logging();
|
||||
let encrypted = objects == RedriveObjects::SseS3OnEtagComputingTarget;
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "matrix-mint-own-redrive-dst".to_string();
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
target.assign_own_version_ids(true);
|
||||
target.ignore_source_etags(encrypted);
|
||||
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
@@ -291,6 +350,7 @@ async fn matrix_mint_own_version_ids_redrive_does_not_duplicate() -> TestResult
|
||||
]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
local_kms: encrypted,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
@@ -319,45 +379,63 @@ async fn matrix_mint_own_version_ids_redrive_does_not_duplicate() -> TestResult
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication(source_env, source_bucket, &target_arn).await?;
|
||||
put_bucket_replication_with_delete_statuses(source_env, source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
|
||||
|
||||
let put = |key: &'static str, fill: u8, len: usize| {
|
||||
let request = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(len, fill)));
|
||||
let request = if encrypted {
|
||||
request.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
} else {
|
||||
request
|
||||
};
|
||||
async move { request.send().await }
|
||||
};
|
||||
|
||||
// Teach the worker the target's identity contract with one ordinary
|
||||
// write, exactly as production learns it (the PUT response carries the
|
||||
// minted id).
|
||||
let probe_key = "redrive/identity-probe.bin";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(probe_key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x01)))
|
||||
.send()
|
||||
.await?;
|
||||
let probe = put(probe_key, 0x01, 4 * 1024).await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, probe_key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
if encrypted {
|
||||
// The variant only proves something while the replica's ETag differs
|
||||
// from the source's encrypted-ETag fingerprint.
|
||||
let replica = single_target_version(&target, &target_bucket, probe_key)?;
|
||||
let replica_etag = fake_source_client(&target)
|
||||
.head_object()
|
||||
.bucket(&target_bucket)
|
||||
.key(probe_key)
|
||||
.version_id(&replica)
|
||||
.send()
|
||||
.await?
|
||||
.e_tag()
|
||||
.map(str::to_string);
|
||||
if replica_etag.is_none() || replica_etag.as_deref() == probe.e_tag() {
|
||||
return Err(format!(
|
||||
"the replica ETag {replica_etag:?} must differ from the source fingerprint {:?}",
|
||||
probe.e_tag()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
|
||||
// Shape 1: the PUT is stored, its response never arrives, heal re-drives.
|
||||
let heal_key = "redrive/heal.bin";
|
||||
target.inject_for_key(FakeTargetOperation::PutObject, heal_key, FakeTargetFault::DisconnectAfterResponse, 1);
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(heal_key)
|
||||
.body(ByteStream::from(payload(8 * 1024, 0x02)))
|
||||
.send()
|
||||
.await?;
|
||||
let healed = put(heal_key, 0x02, 8 * 1024).await?;
|
||||
let healed_version = healed.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
wait_for_replication_status_and_single_version(&source_client, source_bucket, &target, &target_bucket, heal_key).await?;
|
||||
|
||||
// Shape 2: an existing-object resync re-drives a COMPLETED object.
|
||||
let resync_key = "redrive/resync.bin";
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(resync_key)
|
||||
.body(ByteStream::from(payload(8 * 1024, 0x03)))
|
||||
.send()
|
||||
.await?;
|
||||
put(resync_key, 0x03, 8 * 1024).await?;
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, resync_key).await?,
|
||||
"COMPLETED"
|
||||
@@ -388,6 +466,20 @@ async fn matrix_mint_own_version_ids_redrive_does_not_duplicate() -> TestResult
|
||||
);
|
||||
}
|
||||
|
||||
// The heal pass recorded the replica it located; a permanent delete of
|
||||
// that version must remove exactly it.
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(heal_key)
|
||||
.version_id(&healed_version)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("permanent delete of the healed replica", || async {
|
||||
Ok(target.stored_versions(&target_bucket, heal_key).is_empty())
|
||||
})
|
||||
.await?;
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
@@ -871,6 +963,11 @@ async fn matrix_mint_own_version_ids_target() -> TestResult {
|
||||
run_row(TargetMode::MintOwnVersionIds).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn matrix_computes_own_etags_target() -> TestResult {
|
||||
run_row(TargetMode::ComputesOwnEtags).await
|
||||
}
|
||||
|
||||
/// Every known-red entry must name a real cell and an issue, and the lookup
|
||||
/// must round-trip, so a stale or mistyped entry cannot silently pin nothing.
|
||||
#[test]
|
||||
@@ -907,6 +1004,8 @@ async fn run_row(mode: TargetMode) -> TestResult {
|
||||
env_vars.extend_from_slice(&[("NO_PROXY", "127.0.0.1,localhost"), ("HTTP_PROXY", ""), ("HTTPS_PROXY", "")]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
// The SseS3 shape needs a KMS for SSE-S3.
|
||||
local_kms: true,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -31,6 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
gcs = ["dep:google-cloud-storage", "dep:google-cloud-auth"]
|
||||
# Compiles the controlled list-objects namespace-journal chaos injector into a
|
||||
# production binary (it is always available to tests). Off by default so the
|
||||
@@ -249,6 +250,7 @@ windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
aws-smithy-async.workspace = true
|
||||
rustfs-s3-ops.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
criterion = { workspace = true, features = ["html_reports"] }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
|
||||
@@ -14,6 +14,15 @@
|
||||
|
||||
//! Explicit ECStore public facades for outer crate compatibility boundaries.
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub mod diagnostic {
|
||||
pub use crate::cluster::rpc::runtime_sources::{
|
||||
current_read_version_entry_source, observe_read_version_context_boundary, with_read_version_entry,
|
||||
with_read_version_spawn_context,
|
||||
};
|
||||
pub use rustfs_io_metrics::internode_metrics::{ReadVersionContextBoundary, ReadVersionEntrySource};
|
||||
}
|
||||
|
||||
pub mod admin {
|
||||
pub use crate::diagnostics::admin_server_info::{get_local_server_property, get_server_info};
|
||||
}
|
||||
@@ -423,13 +432,14 @@ pub mod disk {
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction,
|
||||
DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation,
|
||||
DeleteRecoveryRpcReply, DeleteRecoveryState, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
|
||||
DiskStore, FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp,
|
||||
ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
is_quota_mutation_fence_path, new_disk, validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
@@ -498,17 +508,19 @@ pub mod metrics {
|
||||
}
|
||||
|
||||
pub mod notification {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::rotate_cross_pool_fence_fleet_proof_for_test;
|
||||
pub use crate::services::notification_sys::{
|
||||
ClusterTierDailyStats, CrossPoolFenceFleetProofToken, IlmRecoveryExportFleetProofToken,
|
||||
LegacyTransitionStateReconcileFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, acquire_ilm_recovery_export_fleet_proof,
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
ilm_recovery_export_fleet_proof_matches, ilm_recovery_export_local_process_epoch,
|
||||
ilm_recovery_export_member_epochs_sha256, ilm_recovery_export_topology_generation,
|
||||
legacy_transition_state_reconcile_fleet_proof_matches, new_global_notification_sys,
|
||||
scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
acquire_legacy_transition_state_reconcile_fleet_proof, configured_fleet_topology_fingerprint,
|
||||
cross_pool_fence_fleet_proof_matches, get_global_notification_sys, ilm_recovery_export_fleet_proof_matches,
|
||||
ilm_recovery_export_local_process_epoch, ilm_recovery_export_member_epochs_sha256,
|
||||
ilm_recovery_export_topology_generation, legacy_transition_state_reconcile_fleet_proof_matches,
|
||||
new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use crate::services::notification_sys::{
|
||||
rotate_cross_pool_fence_fleet_proof_for_test, without_cross_pool_fence_fleet_proof_for_test,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -142,11 +142,11 @@ fn same_replication_service(edited: &BucketTarget, previous: &BucketTarget) -> b
|
||||
&& access_key(edited) == access_key(previous)
|
||||
}
|
||||
|
||||
/// Page size and page budget for [`TargetClient::locate_replica_by_etag`].
|
||||
/// Page size and page budget for [`TargetClient::replica_candidates`].
|
||||
const FIND_VERSION_BY_ETAG_PAGE_SIZE: i32 = 1000;
|
||||
const FIND_VERSION_BY_ETAG_MAX_PAGES: usize = 8;
|
||||
/// Candidate cap for [`TargetClient::replica_candidates_by_etag`]: more than
|
||||
/// this many same-content versions of one key is ambiguity by any measure.
|
||||
/// Candidate cap for [`TargetClient::replica_candidates`]: more than this
|
||||
/// many matching versions of one key is ambiguity by any measure.
|
||||
const FIND_VERSION_BY_ETAG_MAX_MATCHES: usize = 16;
|
||||
pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
|
||||
pub type GetObjectTaggingSdkError = Box<SdkError<GetObjectTaggingError>>;
|
||||
@@ -2030,27 +2030,28 @@ impl TargetClient {
|
||||
|
||||
/// Candidate replicas by content identity on a target that mints its own
|
||||
/// version ids: page `ListObjectVersions` under the exact key and report
|
||||
/// the live versions whose ETag matches `source_etag`, newest first.
|
||||
/// Delete markers and prefix siblings never match. Bounded to
|
||||
/// the live versions that match `identity`, newest first. Delete markers
|
||||
/// and prefix siblings never match. Bounded to
|
||||
/// [`FIND_VERSION_BY_ETAG_MAX_PAGES`] pages and
|
||||
/// [`FIND_VERSION_BY_ETAG_MAX_MATCHES`] candidates so a key with a very
|
||||
/// deep history cannot turn one convergence check into an unbounded scan;
|
||||
/// a replica beyond that window reads as missing, which only costs a
|
||||
/// re-PUT (today's behaviour), never a lost object.
|
||||
/// [`FIND_VERSION_BY_ETAG_MAX_MATCHES`] candidates per kind so a key with
|
||||
/// a very deep history cannot turn one convergence check into an
|
||||
/// unbounded scan; a replica beyond that window reads as missing, which
|
||||
/// only costs a re-PUT (today's behaviour), never a lost object.
|
||||
///
|
||||
/// Content identity is not version identity: two source generations with
|
||||
/// the same bytes have the same ETag. Callers drop the candidates other
|
||||
/// source versions already claim through their ledgers and refuse an
|
||||
/// [`ReplicaLocation::Ambiguous`] remainder before mutating or deleting.
|
||||
pub async fn replica_candidates_by_etag(
|
||||
/// the same bytes have the same ETag, and two with the same size have the
|
||||
/// same shape. Callers drop the candidates other source versions already
|
||||
/// claim through their ledgers and decide through
|
||||
/// [`ReplicaCandidates::locate`].
|
||||
pub async fn replica_candidates(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
source_etag: &str,
|
||||
) -> Result<Vec<String>, Box<SdkError<aws_sdk_s3::operation::list_object_versions::ListObjectVersionsError>>> {
|
||||
identity: &ReplicaIdentity<'_>,
|
||||
) -> Result<ReplicaCandidates, Box<SdkError<aws_sdk_s3::operation::list_object_versions::ListObjectVersionsError>>> {
|
||||
let mut key_marker: Option<String> = None;
|
||||
let mut version_id_marker: Option<String> = None;
|
||||
let mut matches: Vec<String> = Vec::new();
|
||||
let mut candidates = ReplicaCandidates::default();
|
||||
for _ in 0..FIND_VERSION_BY_ETAG_MAX_PAGES {
|
||||
let page = self
|
||||
.client
|
||||
@@ -2063,19 +2064,27 @@ impl TargetClient {
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)?;
|
||||
matches.extend(
|
||||
page.versions()
|
||||
.iter()
|
||||
.filter(|version| {
|
||||
version.key() == Some(object)
|
||||
&& version.version_id().is_some_and(|id| !id.is_empty())
|
||||
&& replication_etags_match(Some(source_etag), version.e_tag())
|
||||
})
|
||||
.filter_map(|version| version.version_id().map(str::to_string)),
|
||||
);
|
||||
for version in page.versions() {
|
||||
let Some(version_id) = version.version_id().filter(|id| !id.is_empty()) else {
|
||||
continue;
|
||||
};
|
||||
if version.key() != Some(object) {
|
||||
continue;
|
||||
}
|
||||
match identity.evidence(version.e_tag(), version.size(), version.last_modified()) {
|
||||
Some(ReplicaEvidence::Etag) => candidates.exact.push(version_id.to_string()),
|
||||
// An unversioned target overwrites the key in place: its
|
||||
// `null` version holds whichever generation was written
|
||||
// last, so a shape match there identifies nothing.
|
||||
Some(ReplicaEvidence::Shape) if version_id != "null" => candidates.shaped.push(version_id.to_string()),
|
||||
Some(ReplicaEvidence::Shape) => {}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
// A listing that moved past the exact key (every listed key is >=
|
||||
// the prefix), ended, or already filled the candidate cap decides.
|
||||
if matches.len() >= FIND_VERSION_BY_ETAG_MAX_MATCHES
|
||||
// the prefix), ended, or already filled a candidate cap decides.
|
||||
if candidates.exact.len() >= FIND_VERSION_BY_ETAG_MAX_MATCHES
|
||||
|| candidates.shaped.len() >= FIND_VERSION_BY_ETAG_MAX_MATCHES
|
||||
|| page
|
||||
.versions()
|
||||
.iter()
|
||||
@@ -2090,8 +2099,9 @@ impl TargetClient {
|
||||
break;
|
||||
}
|
||||
}
|
||||
matches.truncate(FIND_VERSION_BY_ETAG_MAX_MATCHES);
|
||||
Ok(matches)
|
||||
candidates.exact.truncate(FIND_VERSION_BY_ETAG_MAX_MATCHES);
|
||||
candidates.shaped.truncate(FIND_VERSION_BY_ETAG_MAX_MATCHES);
|
||||
Ok(candidates)
|
||||
}
|
||||
|
||||
/// PutObjectRetention against a replica version on a target that does not
|
||||
@@ -2614,19 +2624,18 @@ impl TargetClient {
|
||||
}
|
||||
|
||||
/// Where a replica stands on a target that mints its own version ids, by
|
||||
/// content identity (exact key + ETag) after the candidates other source
|
||||
/// versions claim were removed. See
|
||||
/// [`TargetClient::replica_candidates_by_etag`].
|
||||
/// content identity after the candidates other source versions claim were
|
||||
/// removed. See [`TargetClient::replica_candidates`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ReplicaLocation {
|
||||
/// No live version under the key carries the source ETag.
|
||||
/// No live version under the key matches the source.
|
||||
Missing,
|
||||
/// Exactly one live version carries it: safe to address.
|
||||
/// Exactly one live version matches: safe to address.
|
||||
Unique(String),
|
||||
/// More than one live version carries it (same bytes replicated for
|
||||
/// several source generations). `newest` is the most recently listed
|
||||
/// one — good enough to prove the replica exists, never good enough to
|
||||
/// pick which one to mutate or delete.
|
||||
/// More than one live version carries the source ETag (same bytes
|
||||
/// replicated for several source generations). `newest` is the most
|
||||
/// recently listed one — good enough to prove the replica exists, never
|
||||
/// good enough to pick which one to mutate or delete.
|
||||
Ambiguous { newest: String },
|
||||
}
|
||||
|
||||
@@ -2652,6 +2661,107 @@ impl ReplicaLocation {
|
||||
}
|
||||
}
|
||||
|
||||
/// What a replica of an encrypted source version still has in common with
|
||||
/// it on a target that computes its own ETags. The source ETag is a keyed
|
||||
/// fingerprint (docs/operations/encrypted-etags.md) or a legacy plaintext
|
||||
/// MD5 the target's re-encrypted replica never reports, so the replica is
|
||||
/// recognised by the byte count the replication PUT sent and by a write time
|
||||
/// no earlier than the source version's own.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct EncryptedReplicaShape {
|
||||
/// Bytes on the replication wire: the plaintext size for managed SSE,
|
||||
/// the stored ciphertext size for an SSE-C passthrough.
|
||||
pub size: i64,
|
||||
/// The source version's mtime in whole seconds; listings and HEAD report
|
||||
/// second-precision times.
|
||||
pub not_before_secs: i64,
|
||||
}
|
||||
|
||||
impl EncryptedReplicaShape {
|
||||
pub fn matches(&self, size: Option<i64>, last_modified: Option<&aws_sdk_s3::primitives::DateTime>) -> bool {
|
||||
size == Some(self.size) && last_modified.is_some_and(|written| written.secs() >= self.not_before_secs)
|
||||
}
|
||||
}
|
||||
|
||||
/// How a source version recognises its replica among a key's target
|
||||
/// versions.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReplicaIdentity<'a> {
|
||||
/// Unencrypted source: its ETag is the MD5 of the replicated bytes, which
|
||||
/// every target reports back — exact identity.
|
||||
Etag(&'a str),
|
||||
/// Encrypted source: a target that adopts the source ETag (RustFS,
|
||||
/// MinIO) still reports `etag`; any other target only keeps the shape.
|
||||
Encrypted {
|
||||
etag: Option<&'a str>,
|
||||
shape: EncryptedReplicaShape,
|
||||
},
|
||||
}
|
||||
|
||||
/// Which kind of evidence tied a target version to the source.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReplicaEvidence {
|
||||
/// The target reports the source ETag.
|
||||
Etag,
|
||||
/// Encrypted source on a target that computes its own ETag: same size,
|
||||
/// written no earlier than the source version.
|
||||
Shape,
|
||||
}
|
||||
|
||||
impl ReplicaIdentity<'_> {
|
||||
pub fn evidence(
|
||||
&self,
|
||||
etag: Option<&str>,
|
||||
size: Option<i64>,
|
||||
last_modified: Option<&aws_sdk_s3::primitives::DateTime>,
|
||||
) -> Option<ReplicaEvidence> {
|
||||
match *self {
|
||||
Self::Etag(source_etag) => replication_etags_match(Some(source_etag), etag).then_some(ReplicaEvidence::Etag),
|
||||
Self::Encrypted {
|
||||
etag: source_etag,
|
||||
shape,
|
||||
} => {
|
||||
if replication_etags_match(source_etag, etag) {
|
||||
Some(ReplicaEvidence::Etag)
|
||||
} else {
|
||||
shape.matches(size, last_modified).then_some(ReplicaEvidence::Shape)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Content-identity candidates of one key, newest first, by evidence kind.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ReplicaCandidates {
|
||||
/// Versions that report the source ETag.
|
||||
pub exact: Vec<String>,
|
||||
/// Encrypted sources only: versions without the source ETag whose shape
|
||||
/// matches.
|
||||
pub shaped: Vec<String>,
|
||||
}
|
||||
|
||||
impl ReplicaCandidates {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.exact.is_empty() && self.shaped.is_empty()
|
||||
}
|
||||
|
||||
/// Drop the candidates other source versions claim and decide. An ETag
|
||||
/// match outranks any shape match. A shape is weaker evidence than an
|
||||
/// ETag — two generations of one size share it — so several shape
|
||||
/// matches are never resolved by picking one: they read as
|
||||
/// [`ReplicaLocation::Missing`] for convergence (the caller re-PUTs, a
|
||||
/// duplicate at worst) and as ambiguous for a purge (it refuses).
|
||||
pub fn locate(mut self, claimed: &HashSet<String>) -> (ReplicaLocation, ReplicaEvidence) {
|
||||
self.exact.retain(|candidate| !claimed.contains(candidate));
|
||||
if !self.exact.is_empty() {
|
||||
return (ReplicaLocation::from_matches(self.exact), ReplicaEvidence::Etag);
|
||||
}
|
||||
self.shaped.retain(|candidate| !claimed.contains(candidate));
|
||||
(ReplicaLocation::from_matches(self.shaped), ReplicaEvidence::Shape)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum BucketTargetError {
|
||||
BucketRemoteTargetNotFound {
|
||||
|
||||
@@ -2461,7 +2461,13 @@ fn spawn_recovery_disposition_maintenance_once(api: Arc<ECStore>) -> Option<Join
|
||||
return None;
|
||||
}
|
||||
Some(tokio::spawn(async move {
|
||||
run_recovery_disposition_maintenance_loop(api, cancel_token).await;
|
||||
let maintenance = run_recovery_disposition_maintenance_loop(api, cancel_token);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let maintenance = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::RecoveryDispositionMaintenance,
|
||||
maintenance,
|
||||
);
|
||||
maintenance.await;
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -3344,7 +3350,13 @@ fn spawn_tier_delete_journal_recovery_once(api: Arc<ECStore>) {
|
||||
return;
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
run_tier_delete_journal_recovery_loop(api, cancel_token).await;
|
||||
let recovery = run_tier_delete_journal_recovery_loop(api, cancel_token);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let recovery = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::LifecycleRecovery,
|
||||
recovery,
|
||||
);
|
||||
recovery.await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3363,7 +3375,13 @@ fn spawn_transition_transaction_recovery_once(api: Arc<ECStore>) {
|
||||
return;
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
run_transition_transaction_recovery_loop(api, cancel_token).await;
|
||||
let recovery = run_transition_transaction_recovery_loop(api, cancel_token);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let recovery = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::LifecycleRecovery,
|
||||
recovery,
|
||||
);
|
||||
recovery.await;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3948,44 +3966,28 @@ pub(crate) async fn enqueue_immediate_expiry(api: Arc<ECStore>, oi: &ObjectInfo,
|
||||
return;
|
||||
};
|
||||
|
||||
let mut marker = None;
|
||||
let mut version_marker = None;
|
||||
let mut object_infos = Vec::new();
|
||||
|
||||
loop {
|
||||
let page = match api
|
||||
.clone()
|
||||
.list_object_versions_for_lifecycle(&oi.bucket, &oi.name, marker.clone(), version_marker.clone(), None, 1000)
|
||||
.await
|
||||
{
|
||||
Ok(page) => page,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_EVALUATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
error = %err,
|
||||
reason = "list_versions_failed",
|
||||
"Failed to load lifecycle version group"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
object_infos.extend(page.objects.into_iter().filter(|object| object.name == oi.name));
|
||||
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
let object_infos = match api
|
||||
.get_object_versions_for_lifecycle(&oi.bucket, &oi.name, opts, configs.bucket_incarnation_id)
|
||||
.await
|
||||
{
|
||||
Ok(objects) => objects,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_EVALUATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
result = "failed",
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
error = %err,
|
||||
reason = "read_versions_failed",
|
||||
"Failed to load lifecycle version group"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
marker = page.next_marker;
|
||||
version_marker = page.next_version_idmarker;
|
||||
}
|
||||
|
||||
};
|
||||
if object_infos.is_empty() {
|
||||
object_infos.push(oi.clone());
|
||||
return;
|
||||
}
|
||||
|
||||
let object_opts = object_infos
|
||||
@@ -13203,6 +13205,81 @@ mod tests {
|
||||
assert_eq!(event.action, IlmAction::NoneAction);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn lifecycle_exact_immediate_expiry_preserves_guards_and_skips_stale_writes() {
|
||||
let (paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("lifecycle-exact-expiry-{}", Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
let oi = ecstore
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(b"payload".to_vec()),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("create object before configuring expiry");
|
||||
|
||||
// Exercise persisted legacy immediate rules without relaxing the S3
|
||||
// validator, which correctly rejects newly submitted Expiration.Days=0.
|
||||
let sys = metadata_sys::bucket_metadata_sys_of(&ecstore.ctx).expect("metadata system");
|
||||
let sys = sys.read().await.clone();
|
||||
let mut metadata = (*sys.get(&bucket).await.expect("bucket metadata")).clone();
|
||||
let mut lifecycle = latest_expiration_lifecycle();
|
||||
lifecycle.rules[0].expiration.as_mut().expect("expiration rule").days = Some(0);
|
||||
metadata.lifecycle_config_xml = crate::bucket::utils::serialize(&lifecycle).expect("encode legacy policy");
|
||||
metadata.lifecycle_config = Some(lifecycle);
|
||||
sys.persist_and_set(metadata).await.expect("persist legacy policy");
|
||||
|
||||
let state = install_unconsumed_runtime_expiry_worker(&ecstore, 4).await;
|
||||
super::enqueue_immediate_expiry(ecstore.clone(), &oi, LcEventSrc::S3PutObject, &ObjectOptions::default()).await;
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 1, "exact reads must preserve immediate expiry");
|
||||
|
||||
let state = install_unconsumed_runtime_expiry_worker(&ecstore, 4).await;
|
||||
for path in &paths {
|
||||
let metadata_path = path.join(&bucket).join(object).join(crate::disk::STORAGE_FORMAT_FILE);
|
||||
let mut meta = rustfs_filemeta::FileMeta::load(&fs::read(&metadata_path).await.expect("read object metadata"))
|
||||
.expect("decode object metadata");
|
||||
let mut version = meta
|
||||
.get_all_file_info_versions(&bucket, object, true)
|
||||
.expect("version group")
|
||||
.versions
|
||||
.remove(0);
|
||||
version.metadata.insert("x-amz-object-lock-legal-hold".into(), "ON".into());
|
||||
meta.add_version(version).expect("persist legal hold");
|
||||
fs::write(metadata_path, meta.marshal_msg().expect("encode held metadata"))
|
||||
.await
|
||||
.expect("write legal hold");
|
||||
}
|
||||
super::enqueue_immediate_expiry(ecstore.clone(), &oi, LcEventSrc::S3PutObject, &ObjectOptions::default()).await;
|
||||
assert_eq!(
|
||||
state.read().await.stats.pending_tasks(),
|
||||
0,
|
||||
"use current lock metadata, not the stale write result"
|
||||
);
|
||||
|
||||
for path in &paths {
|
||||
fs::write(path.join(&bucket).join(object).join(crate::disk::STORAGE_FORMAT_FILE), b"corrupt")
|
||||
.await
|
||||
.expect("inject corrupt metadata");
|
||||
}
|
||||
super::enqueue_immediate_expiry(ecstore.clone(), &oi, LcEventSrc::S3PutObject, &ObjectOptions::default()).await;
|
||||
assert_eq!(state.read().await.stats.pending_tasks(), 0, "read failure must never enqueue expiry");
|
||||
for path in &paths {
|
||||
fs::remove_dir_all(path.join(&bucket).join(object))
|
||||
.await
|
||||
.expect("simulate concurrent object removal");
|
||||
}
|
||||
super::enqueue_immediate_expiry(ecstore.clone(), &oi, LcEventSrc::S3PutObject, &ObjectOptions::default()).await;
|
||||
assert_eq!(
|
||||
state.read().await.stats.pending_tasks(),
|
||||
0,
|
||||
"confirmed absence must not fall back to the old ObjectInfo"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn lifecycle_expiry_fails_closed_on_corrupt_object_lock_metadata() {
|
||||
|
||||
@@ -419,7 +419,6 @@ pub(crate) struct QuotaContext {
|
||||
quota_revision: Option<OffsetDateTime>,
|
||||
quota_limit: Option<u64>,
|
||||
reservation_protocol: Option<u32>,
|
||||
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
snapshot_admission: Option<QuotaAdmission>,
|
||||
legacy_data_movement: bool,
|
||||
metadata_guard: Option<Arc<NamespaceLockGuard>>,
|
||||
@@ -479,7 +478,6 @@ impl QuotaContext {
|
||||
reservation: reservation.clone(),
|
||||
};
|
||||
let metadata_guard = self.metadata_guard;
|
||||
let capability_proof = self.capability_proof;
|
||||
|
||||
tokio::spawn(async move {
|
||||
reap_stale_reservations(Arc::clone(&store), &ledger_data.bucket, &ledger_data.ledger_object).await?;
|
||||
@@ -526,7 +524,6 @@ impl QuotaContext {
|
||||
ledger: Some(ReservationLedgerData::Single(ledger_data)),
|
||||
operation_guard: Some(operation_guard),
|
||||
metadata_guard,
|
||||
capability_proof,
|
||||
state: ReservationState::Pending,
|
||||
})
|
||||
})
|
||||
@@ -555,7 +552,6 @@ async fn reserve_sharded(context: QuotaContext, old_size: u64, new_size: u64) ->
|
||||
commit_started: false,
|
||||
};
|
||||
let metadata_guard = context.metadata_guard;
|
||||
let capability_proof = context.capability_proof;
|
||||
let data = ShardedReservationData {
|
||||
store: Arc::clone(&store),
|
||||
bucket: context.bucket,
|
||||
@@ -640,7 +636,6 @@ async fn reserve_sharded(context: QuotaContext, old_size: u64, new_size: u64) ->
|
||||
ledger: Some(ReservationLedgerData::Sharded(data)),
|
||||
operation_guard: Some(operation_guard),
|
||||
metadata_guard,
|
||||
capability_proof,
|
||||
state: ReservationState::Pending,
|
||||
});
|
||||
}
|
||||
@@ -725,20 +720,12 @@ impl ReservationLedgerData {
|
||||
Self::Sharded(data) => &data.bucket,
|
||||
}
|
||||
}
|
||||
|
||||
fn error_object(&self) -> &str {
|
||||
match self {
|
||||
Self::Single(data) => &data.ledger_object,
|
||||
Self::Sharded(data) => &data.shard_object,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct QuotaReservation {
|
||||
ledger: Option<ReservationLedgerData>,
|
||||
operation_guard: Option<NamespaceLockGuard>,
|
||||
metadata_guard: Option<Arc<NamespaceLockGuard>>,
|
||||
capability_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
state: ReservationState,
|
||||
}
|
||||
|
||||
@@ -756,7 +743,6 @@ impl QuotaReservation {
|
||||
ledger: None,
|
||||
operation_guard: None,
|
||||
metadata_guard,
|
||||
capability_proof: None,
|
||||
state: ReservationState::Pending,
|
||||
}
|
||||
}
|
||||
@@ -766,17 +752,7 @@ impl QuotaReservation {
|
||||
|| self.metadata_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
}
|
||||
|
||||
pub(crate) fn capability_proof_matches(&self) -> bool {
|
||||
self.capability_proof
|
||||
.as_ref()
|
||||
.is_none_or(crate::services::notification_sys::cross_pool_fence_fleet_proof_matches)
|
||||
}
|
||||
|
||||
pub(crate) async fn mark_commit_started(&mut self) -> Result<()> {
|
||||
if !self.capability_proof_matches() {
|
||||
let ledger = self.ledger.as_ref().ok_or(StorageError::PartMissingOrCorrupt)?;
|
||||
return Err(quota_capability_error(ledger.bucket(), ledger.error_object()));
|
||||
}
|
||||
if let Some(ledger) = self.ledger.as_ref() {
|
||||
mark_commit_started(ledger).await?;
|
||||
}
|
||||
@@ -864,7 +840,6 @@ pub(crate) async fn begin(
|
||||
quota_revision: None,
|
||||
quota_limit: None,
|
||||
reservation_protocol: None,
|
||||
capability_proof: None,
|
||||
snapshot_admission: None,
|
||||
legacy_data_movement: false,
|
||||
metadata_guard: None,
|
||||
@@ -883,7 +858,6 @@ pub(crate) async fn begin(
|
||||
quota_revision: None,
|
||||
quota_limit: Some(snapshot_admission.quota_limit()),
|
||||
reservation_protocol: None,
|
||||
capability_proof: None,
|
||||
snapshot_admission: Some(snapshot_admission),
|
||||
legacy_data_movement: false,
|
||||
metadata_guard: None,
|
||||
@@ -902,7 +876,6 @@ pub(crate) async fn begin(
|
||||
quota_revision: None,
|
||||
quota_limit: None,
|
||||
reservation_protocol: None,
|
||||
capability_proof: None,
|
||||
snapshot_admission: None,
|
||||
legacy_data_movement: false,
|
||||
metadata_guard: None,
|
||||
@@ -994,19 +967,9 @@ pub(crate) async fn begin(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE,
|
||||
policy_evaluate_started,
|
||||
);
|
||||
let capability_proof_started = rustfs_io_metrics::put_stage_timer();
|
||||
let capability_proof = if durable_quota.is_some() {
|
||||
Some(
|
||||
crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
|
||||
.ok_or_else(|| quota_capability_error(bucket, &ledger_object(bucket)))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF,
|
||||
capability_proof_started,
|
||||
);
|
||||
// Durable writes are fenced by a write quorum of participating disks that
|
||||
// enforce the configured topology. Fleet-wide proof remains a configuration gate;
|
||||
// losing an unrelated peer must not revoke an admitted object write.
|
||||
let durable_quota_limit = durable_quota.and_then(|quota| quota.quota);
|
||||
let snapshot_admission_started = rustfs_io_metrics::put_stage_timer();
|
||||
let reservation_protocol = durable_quota.and_then(|quota| quota.reservation_protocol);
|
||||
@@ -1067,7 +1030,6 @@ pub(crate) async fn begin(
|
||||
quota_revision: Some(quota_revision),
|
||||
quota_limit,
|
||||
reservation_protocol,
|
||||
capability_proof,
|
||||
snapshot_admission,
|
||||
legacy_data_movement,
|
||||
metadata_guard: Some(metadata_guard),
|
||||
@@ -1085,16 +1047,6 @@ fn quota_cache_reason(quota: Option<&crate::bucket::quota::BucketQuota>) -> &'st
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_capability_error(bucket: &str, object: &str) -> StorageError {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_capability",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn replaced_logical_size(set_disks: &SetDisks, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<u64> {
|
||||
if opts.versioned && !opts.version_suspended && opts.version_id.is_none() {
|
||||
return Ok(0);
|
||||
|
||||
@@ -36,9 +36,9 @@ use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
|
||||
delete_replication_creates_marker, delete_replication_target_version_id, heal_uses_delete_replication_path,
|
||||
is_object_lock_denied_delete, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, replication_single_put_size_error, resync_existing_delete_replication_info,
|
||||
should_retry_delete_marker_purge, single_part_replica_etag_mismatch,
|
||||
replicate_delete_outcome, replication_multipart_complete_actual_size, replication_multipart_part_plan,
|
||||
replication_single_put_size_error, resync_existing_delete_replication_info, should_retry_delete_marker_purge,
|
||||
single_part_replica_etag_mismatch,
|
||||
};
|
||||
use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
|
||||
use super::replication_resync_boundary::ResyncStatusType;
|
||||
@@ -60,12 +60,13 @@ use super::replication_storage_boundary::{
|
||||
use super::replication_storage_boundary::{NamespaceLockFence, NamespaceLockSignalTestFence, ReplicationDeletedObject};
|
||||
use super::replication_tagging_boundary::ReplicationTagFilter;
|
||||
use super::replication_target_boundary::{
|
||||
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
|
||||
RemotePutObjectResponse, ReplicaLocation, ReplicationTargetStore, S3ClientError, SsecPassthroughCapability,
|
||||
SsecPassthroughGate, TargetClient, VersionIdentityCapability, is_replication_target_offline_error,
|
||||
replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options,
|
||||
replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted,
|
||||
replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version,
|
||||
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, EncryptedReplicaShape, HeadObjectSdkError, PutObjectOptions,
|
||||
PutObjectPartOptions, RemotePutObjectResponse, ReplicaCandidates, ReplicaEvidence, ReplicaIdentity, ReplicaLocation,
|
||||
ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient,
|
||||
VersionIdentityCapability, is_replication_target_offline_error, replication_action_for_target_head,
|
||||
replication_complete_multipart_options, replication_delete_marker_purge_remove_options, replication_delete_remove_options,
|
||||
replication_force_delete_remove_options, replication_object_is_ssec_encrypted, replication_put_object_header_size,
|
||||
replication_put_object_options, replication_source_etag_is_comparable, replication_target_head_is_newer_null_version,
|
||||
resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate, version_identity_capability_from_put,
|
||||
version_identity_drifted,
|
||||
};
|
||||
@@ -349,15 +350,23 @@ fn replica_etag_verification_enabled() -> bool {
|
||||
/// target that stores a transformed payload (e.g. undecoded `aws-chunked`
|
||||
/// frames, #6853) returns the ETag of what it actually wrote. Reporting
|
||||
/// COMPLETED over such a replica is silent corruption, so a decidable
|
||||
/// mismatch fails the replication instead. An SSE-C ciphertext passthrough
|
||||
/// transfer is exempt: the wire bytes are ciphertext while the source ETag is
|
||||
/// the plaintext MD5, and that path has its own HEAD-back audit.
|
||||
/// mismatch fails the replication instead. Encrypted sources are exempt: an
|
||||
/// SSE-C ciphertext passthrough sends ciphertext while the source ETag is the
|
||||
/// plaintext MD5 (that path has its own HEAD-back audit), and a managed-SSE
|
||||
/// (SSE-S3 / SSE-KMS) source stores a keyed fingerprint, or a legacy plaintext
|
||||
/// MD5, that the ETag of the target's re-encrypted replica can neither confirm
|
||||
/// nor refute. A managed-SSE source forwards no checksum either
|
||||
/// (`replication_put_object_options`), so its replica has no post-PUT
|
||||
/// content audit.
|
||||
fn verify_single_part_replica(
|
||||
object_info: &ObjectInfo,
|
||||
response: &RemotePutObjectResponse,
|
||||
ciphertext_passthrough: bool,
|
||||
) -> std::result::Result<(), std::io::Error> {
|
||||
if ciphertext_passthrough || !replica_etag_verification_enabled() {
|
||||
if ciphertext_passthrough
|
||||
|| !replication_source_etag_is_comparable(&object_info.user_defined)
|
||||
|| !replica_etag_verification_enabled()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if single_part_replica_etag_mismatch(object_info.etag.as_deref(), response.etag.as_deref()) {
|
||||
@@ -482,6 +491,7 @@ async fn head_object_fallback(
|
||||
Ok(head) => Ok(Some(LocatedReplica {
|
||||
version_id: head.version_id.clone(),
|
||||
head,
|
||||
by: LocatedBy::CurrentVersion,
|
||||
})),
|
||||
Err(e) if head_object_not_found(&e) => Ok(None),
|
||||
Err(e) => Err(e),
|
||||
@@ -495,6 +505,63 @@ async fn head_object_fallback(
|
||||
struct LocatedReplica {
|
||||
head: HeadObjectOutput,
|
||||
version_id: Option<String>,
|
||||
by: LocatedBy,
|
||||
}
|
||||
|
||||
/// How [`replica_head_fallback`] reached a replica, which decides what may
|
||||
/// confirm it (see [`located_replica_matches`]).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum LocatedBy {
|
||||
/// The target version the source ledger recorded for this version.
|
||||
Ledger,
|
||||
/// The one candidate content identity left (`ListObjectVersions`).
|
||||
ContentIdentity,
|
||||
/// Whatever version is current under the key (AWS-style fallback):
|
||||
/// nothing ties it to this source version but an ETag match.
|
||||
CurrentVersion,
|
||||
}
|
||||
|
||||
/// How this source version recognises its replica on a target. `None` when
|
||||
/// the source carries nothing to compare (no ETag on a plaintext source).
|
||||
fn replica_identity(object_info: &ObjectInfo) -> Option<ReplicaIdentity<'_>> {
|
||||
let etag = object_info.etag.as_deref().filter(|etag| !etag.trim().is_empty());
|
||||
if replication_source_etag_is_comparable(&object_info.user_defined) {
|
||||
return etag.map(ReplicaIdentity::Etag);
|
||||
}
|
||||
// Same byte count the replication PUT sends: SSE-C passes the stored
|
||||
// ciphertext through, managed SSE sends the decrypted plaintext.
|
||||
let size = if replication_object_is_ssec_encrypted(&object_info.user_defined) {
|
||||
object_info.size
|
||||
} else {
|
||||
object_info.get_actual_size().ok()?
|
||||
};
|
||||
match object_info.mod_time {
|
||||
Some(mod_time) if size >= 0 => Some(ReplicaIdentity::Encrypted {
|
||||
etag,
|
||||
shape: EncryptedReplicaShape {
|
||||
size,
|
||||
not_before_secs: mod_time.unix_timestamp(),
|
||||
},
|
||||
}),
|
||||
_ => etag.map(ReplicaIdentity::Etag),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a replica reached through [`replica_head_fallback`] holds this
|
||||
/// source version. An ETag match always does. For an encrypted source on a
|
||||
/// target that computes its own ETag, the shape does too — but only for a
|
||||
/// replica the ledger or content identity singled out, never for whichever
|
||||
/// version happens to be current. A plaintext source still needs its ETag.
|
||||
fn located_replica_matches(identity: Option<&ReplicaIdentity<'_>>, located: &LocatedReplica) -> bool {
|
||||
let Some(identity) = identity else {
|
||||
return false;
|
||||
};
|
||||
let head = &located.head;
|
||||
match identity.evidence(head.e_tag(), head.content_length(), head.last_modified()) {
|
||||
Some(ReplicaEvidence::Etag) => true,
|
||||
Some(ReplicaEvidence::Shape) => located.by != LocatedBy::CurrentVersion,
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn head_object_not_found(err: &HeadObjectSdkError) -> bool {
|
||||
@@ -576,15 +643,20 @@ fn ledger_target_version_id(user_defined: &HashMap<String, String>, arn: &str) -
|
||||
/// Second look at a replica whose version-addressed HEAD failed, for the two
|
||||
/// target shapes where that failure is not a verdict on the replica:
|
||||
///
|
||||
/// - AWS-style 400/403 (the RustFS uuid is rejected as malformed): HEAD the
|
||||
/// current version without a version id; callers compare ETags.
|
||||
/// - AWS-style 400/403 (the RustFS uuid is rejected as malformed) and
|
||||
/// - 404 on a target that mints its own version ids (the Wasabi shape,
|
||||
/// rustfs/backlog#2340): the source id never existed there. HEAD the id
|
||||
/// the source ledger recorded for this target when it has one; otherwise,
|
||||
/// once the target is known to mint ids, locate the replica by exact key
|
||||
/// and ETag through ListObjectVersions and HEAD the id the target assigned.
|
||||
/// Without this, every heal, MRF retry and existing-object resync re-drive
|
||||
/// PUTs the object again and mints one more target version.
|
||||
/// rustfs/backlog#2340): the source id never existed there.
|
||||
///
|
||||
/// Both HEAD the id the source ledger recorded for this target first. Without
|
||||
/// one, the AWS shape HEADs the current version without a version id
|
||||
/// (callers compare ETags); the Wasabi shape, once the target is known to
|
||||
/// mint ids, locates the replica by exact key and content identity through
|
||||
/// ListObjectVersions and HEADs the id the target assigned. An encrypted
|
||||
/// source takes that listing on the AWS shape too: its ETag never matches a
|
||||
/// target-computed one (docs/operations/encrypted-etags.md), so only a unique
|
||||
/// shape match can stand in for it. Without this, every heal, MRF retry and
|
||||
/// existing-object resync re-drive PUTs the object again and mints one more
|
||||
/// target version.
|
||||
///
|
||||
/// `content_identity` is the sibling-ledger view used to disown candidates
|
||||
/// other source versions already claim; `None` disables content identity
|
||||
@@ -601,7 +673,7 @@ struct ReplicaSource<'a> {
|
||||
bucket: &'a str,
|
||||
object: &'a str,
|
||||
version_id: Option<Uuid>,
|
||||
etag: Option<&'a str>,
|
||||
identity: Option<ReplicaIdentity<'a>>,
|
||||
ledger_version_id: Option<&'a str>,
|
||||
}
|
||||
|
||||
@@ -615,13 +687,11 @@ async fn replica_head_fallback(
|
||||
bucket: source_bucket,
|
||||
object,
|
||||
version_id: source_version_id,
|
||||
etag: source_etag,
|
||||
identity,
|
||||
ledger_version_id,
|
||||
} = source;
|
||||
if is_version_id_format_mismatch(err) {
|
||||
return Some(head_object_fallback(tgt_client, object).await);
|
||||
}
|
||||
if !head_object_not_found(err) {
|
||||
let format_mismatch = is_version_id_format_mismatch(err);
|
||||
if !format_mismatch && !head_object_not_found(err) {
|
||||
return None;
|
||||
}
|
||||
if let Some(ledger_version_id) = ledger_version_id {
|
||||
@@ -633,24 +703,45 @@ async fn replica_head_fallback(
|
||||
return Some(Ok(Some(LocatedReplica {
|
||||
head,
|
||||
version_id: Some(ledger_version_id.to_string()),
|
||||
by: LocatedBy::Ledger,
|
||||
})));
|
||||
}
|
||||
// The recorded version is gone: fall through to content identity
|
||||
// before concluding that the replica is missing.
|
||||
Err(e) if head_object_not_found(&e) => {}
|
||||
Err(e) if head_object_not_found(&e) || is_version_id_format_mismatch(&e) => {}
|
||||
Err(e) => return Some(Err(e)),
|
||||
}
|
||||
}
|
||||
if !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable() {
|
||||
let encrypted = matches!(identity, Some(ReplicaIdentity::Encrypted { .. }));
|
||||
if format_mismatch {
|
||||
if !(encrypted && content_identity.is_some()) {
|
||||
return Some(head_object_fallback(tgt_client, object).await);
|
||||
}
|
||||
} else if !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable() {
|
||||
return None;
|
||||
}
|
||||
let siblings = content_identity?;
|
||||
let etag = source_etag.filter(|etag| !etag.trim().is_empty())?;
|
||||
Some(match tgt_client.replica_candidates_by_etag(&tgt_client.bucket, object, etag).await {
|
||||
let identity = identity?;
|
||||
let candidates = match tgt_client.replica_candidates(&tgt_client.bucket, object, &identity).await {
|
||||
Ok(candidates) => Ok(candidates),
|
||||
// A target that refused the version-addressed HEAD (a bodiless 403
|
||||
// can be a permission denial) may refuse the listing too: keep the
|
||||
// current-version fallback, which re-PUTs, instead of stalling.
|
||||
Err(_) if format_mismatch => return Some(head_object_fallback(tgt_client, object).await),
|
||||
Err(list_err) => Err(list_err),
|
||||
};
|
||||
Some(match candidates {
|
||||
Ok(candidates) => {
|
||||
let location =
|
||||
let (location, evidence) =
|
||||
disowned_replica_location(siblings, &tgt_client.arn, source_bucket, object, source_version_id, candidates).await;
|
||||
match location.any_version_id() {
|
||||
// Several same-ETag versions all hold the source bytes, so any of
|
||||
// them proves the replica exists. Several same-shape versions do
|
||||
// not: re-PUT rather than adopt the wrong generation.
|
||||
let assigned_version_id = match (&location, evidence) {
|
||||
(ReplicaLocation::Ambiguous { .. }, ReplicaEvidence::Shape) => None,
|
||||
_ => location.any_version_id(),
|
||||
};
|
||||
match assigned_version_id {
|
||||
Some(assigned_version_id) => {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_DRIFTED_REPLICA_LOCATED,
|
||||
@@ -661,6 +752,7 @@ async fn replica_head_fallback(
|
||||
arn = %tgt_client.arn,
|
||||
assigned_version_id = %assigned_version_id,
|
||||
ambiguous = matches!(location, ReplicaLocation::Ambiguous { .. }),
|
||||
evidence = ?evidence,
|
||||
"Located replica by content identity on a target that mints its own version ids"
|
||||
);
|
||||
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, Some(assigned_version_id.to_string()))
|
||||
@@ -669,6 +761,7 @@ async fn replica_head_fallback(
|
||||
Ok(head) => Ok(Some(LocatedReplica {
|
||||
head,
|
||||
version_id: Some(assigned_version_id.to_string()),
|
||||
by: LocatedBy::ContentIdentity,
|
||||
})),
|
||||
// The located version disappeared between LIST and HEAD.
|
||||
Err(e) if head_object_not_found(&e) => Ok(None),
|
||||
@@ -691,16 +784,15 @@ async fn disowned_replica_location(
|
||||
source_bucket: &str,
|
||||
object: &str,
|
||||
source_version_id: Option<Uuid>,
|
||||
mut candidates: Vec<String>,
|
||||
) -> ReplicaLocation {
|
||||
candidates: ReplicaCandidates,
|
||||
) -> (ReplicaLocation, ReplicaEvidence) {
|
||||
if candidates.is_empty() {
|
||||
return ReplicaLocation::Missing;
|
||||
return (ReplicaLocation::Missing, ReplicaEvidence::Etag);
|
||||
}
|
||||
let claimed = siblings
|
||||
.claimed_target_versions(source_bucket, object, source_version_id, arn)
|
||||
.await;
|
||||
candidates.retain(|candidate| !claimed.contains(candidate));
|
||||
ReplicaLocation::from_matches(candidates)
|
||||
candidates.locate(&claimed)
|
||||
}
|
||||
|
||||
/// The id the target assigned to a located replica when it is not the source
|
||||
@@ -1847,10 +1939,14 @@ fn spawn_resync_walk_task<S: ReplicationStorage>(
|
||||
|
||||
/// Classify the target HEAD verification result for one resynced object,
|
||||
/// updating the per-object status counters and returning the accounted size
|
||||
/// together with any verification error.
|
||||
/// together with any verification error. `source` is the source version as
|
||||
/// stored after the resync replicated it (its ledger and encryption decide
|
||||
/// how the replica is recognised); without it the queued ETag is all there
|
||||
/// is to go on.
|
||||
async fn verify_resync_head_result(
|
||||
head_result: std::result::Result<HeadObjectOutput, HeadObjectSdkError>,
|
||||
roi: &ReplicateObjectInfo,
|
||||
source: Option<&ObjectInfo>,
|
||||
st: &mut TargetReplicationResyncStatus,
|
||||
target_client: &Arc<TargetClient>,
|
||||
) -> (i64, Option<HeadObjectSdkError>) {
|
||||
@@ -1892,14 +1988,23 @@ async fn verify_resync_head_result(
|
||||
// A version-addressed HEAD is not the last word on every target:
|
||||
// re-verify through the fallback before counting a well-replicated
|
||||
// object as failed (see `replica_head_fallback`).
|
||||
let ledger_version_id = source.and_then(|info| ledger_target_version_id(&info.user_defined, &target_client.arn));
|
||||
let identity = match source {
|
||||
Some(info) => replica_identity(info),
|
||||
None => roi
|
||||
.etag
|
||||
.as_deref()
|
||||
.filter(|etag| !etag.trim().is_empty())
|
||||
.map(ReplicaIdentity::Etag),
|
||||
};
|
||||
match replica_head_fallback(
|
||||
target_client.as_ref(),
|
||||
ReplicaSource {
|
||||
bucket: &roi.bucket,
|
||||
object: &roi.name,
|
||||
version_id: roi.version_id,
|
||||
etag: roi.etag.as_deref(),
|
||||
ledger_version_id: None,
|
||||
identity,
|
||||
ledger_version_id: ledger_version_id.as_deref(),
|
||||
},
|
||||
Some(&NoSiblingLedger),
|
||||
&err,
|
||||
@@ -1924,6 +2029,24 @@ async fn verify_resync_head_result(
|
||||
}
|
||||
}
|
||||
|
||||
/// The resynced source version as stored now, ledger included, for the
|
||||
/// verification fallback. `None` when it cannot be read.
|
||||
async fn resync_source_object_info<S: ReplicationStorage>(storage: &Arc<S>, roi: &ReplicateObjectInfo) -> Option<ObjectInfo> {
|
||||
storage
|
||||
.get_object_info(
|
||||
&roi.bucket,
|
||||
&roi.name,
|
||||
&ObjectOptions {
|
||||
version_id: roi.version_id.map(|version_id| version_id.to_string()),
|
||||
versioned: ReplicationVersioningStore::prefix_enabled(&roi.bucket, &roi.name).await,
|
||||
version_suspended: ReplicationVersioningStore::prefix_suspended(&roi.bucket, &roi.name).await,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Replicate one existing object (or delete marker / version purge) to the
|
||||
/// resync target, verify the outcome via a target HEAD, and produce the
|
||||
/// per-object resync status update.
|
||||
@@ -1958,7 +2081,12 @@ async fn resync_worker_process_object<S: ReplicationStorage>(
|
||||
roi.version_id.map(|v| v.to_string()),
|
||||
)
|
||||
.await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, target_client).await;
|
||||
let source = if head_result.is_err() && !roi.delete_marker {
|
||||
resync_source_object_info(storage, &roi).await
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, source.as_ref(), &mut st, target_client).await;
|
||||
|
||||
if err.is_some() {
|
||||
debug!(
|
||||
@@ -3355,6 +3483,9 @@ fn abandoned_purge_target_info(bucket: &str, dobj: &DeletedObjectReplicationInfo
|
||||
/// to resolve the replica on a target that mints its own version ids.
|
||||
struct VersionPurgeSource {
|
||||
etag: Option<String>,
|
||||
/// Encrypted sources: what identifies the replica when the target
|
||||
/// computes its own ETag (see [`replica_identity`]).
|
||||
encrypted_shape: Option<EncryptedReplicaShape>,
|
||||
ledger: HashMap<String, String>,
|
||||
ledger_corrupt: bool,
|
||||
/// Target versions other generations of the key claim, per target ARN:
|
||||
@@ -3400,8 +3531,13 @@ async fn version_purge_source<S: ReplicationStorage>(
|
||||
claimed_by_siblings.insert(target.arn.clone(), claimed);
|
||||
}
|
||||
}
|
||||
let encrypted_shape = match replica_identity(&info) {
|
||||
Some(ReplicaIdentity::Encrypted { shape, .. }) => Some(shape),
|
||||
_ => None,
|
||||
};
|
||||
Some(VersionPurgeSource {
|
||||
etag: info.etag,
|
||||
encrypted_shape,
|
||||
ledger,
|
||||
ledger_corrupt,
|
||||
claimed_by_siblings,
|
||||
@@ -3419,7 +3555,8 @@ enum PurgeReplicaResolution {
|
||||
/// goal is already met there.
|
||||
Absent,
|
||||
/// Refuse: no ledger entry and more than one target version carries the
|
||||
/// source ETag, so any pick could destroy a live generation.
|
||||
/// source ETag (or, for an encrypted source on a target that computes its
|
||||
/// own ETag, its shape), so any pick could destroy a live generation.
|
||||
Ambiguous,
|
||||
/// Refuse: the persisted ledger is inconsistent.
|
||||
LedgerCorrupt,
|
||||
@@ -3447,15 +3584,19 @@ async fn resolve_purge_replica(
|
||||
if !force_lookup && !ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable() {
|
||||
return PurgeReplicaResolution::SourceVersion;
|
||||
}
|
||||
let Some(etag) = source.etag.as_deref().filter(|etag| !etag.trim().is_empty()) else {
|
||||
return PurgeReplicaResolution::SourceVersion;
|
||||
let etag = source.etag.as_deref().filter(|etag| !etag.trim().is_empty());
|
||||
let identity = match (source.encrypted_shape, etag) {
|
||||
(Some(shape), etag) => ReplicaIdentity::Encrypted { etag, shape },
|
||||
(None, Some(etag)) => ReplicaIdentity::Etag(etag),
|
||||
(None, None) => return PurgeReplicaResolution::SourceVersion,
|
||||
};
|
||||
match tgt_client.replica_candidates_by_etag(&tgt_client.bucket, object, etag).await {
|
||||
Ok(mut candidates) => {
|
||||
if let Some(claimed) = source.claimed_by_siblings.get(&tgt_client.arn) {
|
||||
candidates.retain(|candidate| !claimed.contains(candidate));
|
||||
}
|
||||
match ReplicaLocation::from_matches(candidates) {
|
||||
match tgt_client.replica_candidates(&tgt_client.bucket, object, &identity).await {
|
||||
Ok(candidates) => {
|
||||
let no_claims = HashSet::new();
|
||||
let claimed = source.claimed_by_siblings.get(&tgt_client.arn).unwrap_or(&no_claims);
|
||||
// Several shape matches stay ambiguous here: a purge never picks
|
||||
// one to delete.
|
||||
match candidates.locate(claimed).0 {
|
||||
ReplicaLocation::Unique(assigned) => PurgeReplicaResolution::Resolved(assigned),
|
||||
ReplicaLocation::Missing => PurgeReplicaResolution::Absent,
|
||||
ReplicaLocation::Ambiguous { .. } => PurgeReplicaResolution::Ambiguous,
|
||||
@@ -4509,13 +4650,14 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
// generation with the same bytes would be mistaken for one.
|
||||
// Only the ledger (a lost response of this very version) and
|
||||
// the AWS-style format fallback apply here.
|
||||
let identity = replica_identity(&object_info);
|
||||
if let Some(fallback) = replica_head_fallback(
|
||||
&tgt_client,
|
||||
ReplicaSource {
|
||||
bucket: &bucket,
|
||||
object: &object,
|
||||
version_id: self.version_id,
|
||||
etag: object_info.etag.as_deref(),
|
||||
identity,
|
||||
ledger_version_id: ledger_version_id.as_deref(),
|
||||
},
|
||||
None,
|
||||
@@ -4524,9 +4666,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
.await
|
||||
{
|
||||
match fallback {
|
||||
Ok(Some(located))
|
||||
if replication_etags_match(object_info.etag.as_deref(), located.head.e_tag.as_deref()) =>
|
||||
{
|
||||
Ok(Some(located)) if located_replica_matches(identity.as_ref(), &located) => {
|
||||
if ssec_audit_required
|
||||
&& !settle_ssec_passthrough_evidence(&located.head, &tgt_client, &bucket, &object, &mut rinfo)
|
||||
.await
|
||||
@@ -5174,13 +5314,14 @@ async fn resolve_replicate_all_action(
|
||||
}
|
||||
Err(e) => {
|
||||
let ledger_version_id = ledger_target_version_id(&object_info.user_defined, &tgt_client.arn);
|
||||
let identity = replica_identity(&object_info);
|
||||
if let Some(fallback) = replica_head_fallback(
|
||||
tgt_client,
|
||||
ReplicaSource {
|
||||
bucket,
|
||||
object,
|
||||
version_id: roi.version_id,
|
||||
etag: object_info.etag.as_deref(),
|
||||
identity,
|
||||
ledger_version_id: ledger_version_id.as_deref(),
|
||||
},
|
||||
Some(siblings),
|
||||
@@ -5191,20 +5332,20 @@ async fn resolve_replicate_all_action(
|
||||
match fallback {
|
||||
Ok(Some(located)) => {
|
||||
let oi = &located.head;
|
||||
let etags_match = replication_etags_match(object_info.etag.as_deref(), oi.e_tag.as_deref());
|
||||
if require_existing_target && !etags_match {
|
||||
let replica_matches = located_replica_matches(identity.as_ref(), &located);
|
||||
if require_existing_target && !replica_matches {
|
||||
rinfo.error = Some("replica metadata target does not contain matching object data".to_string());
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return None;
|
||||
}
|
||||
if etags_match
|
||||
if replica_matches
|
||||
&& ssec_audit_required
|
||||
&& !settle_ssec_passthrough_evidence(oi, tgt_client, bucket, object, rinfo).await
|
||||
{
|
||||
rinfo.duration = (OffsetDateTime::now_utc() - start_time).unsigned_abs();
|
||||
return None;
|
||||
}
|
||||
if etags_match && let Some(assigned) = drifted_replica_version_id(&located, roi.version_id) {
|
||||
if replica_matches && let Some(assigned) = drifted_replica_version_id(&located, roi.version_id) {
|
||||
// The replica lives under an id the target minted:
|
||||
// record it (the ledger every later mutation
|
||||
// resolves through) and apply metadata in place —
|
||||
@@ -5237,7 +5378,7 @@ async fn resolve_replicate_all_action(
|
||||
// records the sync without running the PUT.
|
||||
replication_action = ReplicationAction::None;
|
||||
} else {
|
||||
replication_action = if etags_match {
|
||||
replication_action = if replica_matches {
|
||||
ReplicationAction::None
|
||||
} else {
|
||||
ReplicationAction::All
|
||||
@@ -5870,6 +6011,57 @@ mod tests {
|
||||
use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
use super::super::replication_object_decision_boundary::ReplicationMultipartPlanError;
|
||||
|
||||
#[test]
|
||||
fn verify_single_part_replica_exempts_managed_sse_sources() {
|
||||
// A new encrypted write stores a keyed fingerprint; a target that
|
||||
// computes its own ETag answers with the MD5 of the plaintext it
|
||||
// received. Both are 32 hex characters, so without the exemption the
|
||||
// audit would fail every SSE-S3 / SSE-KMS replica (rustfs/backlog).
|
||||
let fingerprint = "00112233445566778899aabbccddeeff";
|
||||
let replica_md5 = "ffeeddccbbaa99887766554433221100";
|
||||
let response = RemotePutObjectResponse {
|
||||
version_id: None,
|
||||
etag: Some(replica_md5.to_string()),
|
||||
};
|
||||
for sse in ["AES256", "aws:kms"] {
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "encrypted".to_string(),
|
||||
etag: Some(fingerprint.to_string()),
|
||||
user_defined: HashMap::from([("x-amz-server-side-encryption".to_string(), sse.to_string())]).into(),
|
||||
..Default::default()
|
||||
};
|
||||
verify_single_part_replica(&object_info, &response, false)
|
||||
.unwrap_or_else(|err| panic!("{sse}: a managed-SSE source must not be audited against the replica MD5: {err}"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_single_part_replica_still_fails_plaintext_mismatch() {
|
||||
if !replica_etag_verification_enabled() {
|
||||
return;
|
||||
}
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "plain".to_string(),
|
||||
etag: Some("00112233445566778899aabbccddeeff".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let mismatch = RemotePutObjectResponse {
|
||||
version_id: None,
|
||||
etag: Some("ffeeddccbbaa99887766554433221100".to_string()),
|
||||
};
|
||||
let err = verify_single_part_replica(&object_info, &mismatch, false)
|
||||
.expect_err("a plaintext source whose replica MD5 differs must fail the replication");
|
||||
assert_eq!(err.to_string(), REPLICA_ETAG_MISMATCH_ERROR);
|
||||
|
||||
let matching = RemotePutObjectResponse {
|
||||
version_id: None,
|
||||
etag: object_info.etag.clone(),
|
||||
};
|
||||
verify_single_part_replica(&object_info, &matching, false).expect("matching plaintext MD5s pass the audit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_read_plan_preserves_legacy_plain_part_ranges() {
|
||||
const MIB: usize = 1024 * 1024;
|
||||
@@ -6596,13 +6788,361 @@ mod tests {
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, None, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_none(), "{err:?}");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (4, 1, 0));
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
/// What a target that computes its own ETag (AWS S3) reports for the
|
||||
/// replica of an encrypted source: the MD5 of the plaintext it received,
|
||||
/// never the source's encrypted-ETag fingerprint ([`DRIFTED_ETAG`] here).
|
||||
const COMPUTED_REPLICA_ETAG: &str = "0f0e0d0c0b0a09080706050403020100";
|
||||
/// The replica's write time on the scripted targets below.
|
||||
const REPLICA_WRITTEN: &str = "2026-09-06T10:00:00.000Z";
|
||||
const REPLICA_WRITTEN_HTTP: &str = "Sun, 06 Sep 2026 10:00:00 GMT";
|
||||
|
||||
/// An SSE-S3 source version: 4 plaintext bytes, stored as ciphertext, its
|
||||
/// ETag a keyed fingerprint, written at `written` (unix seconds).
|
||||
fn encrypted_roi_and_object(written: i64) -> (ReplicateObjectInfo, ObjectInfo) {
|
||||
let (roi, mut object_info) = drifted_roi_and_object();
|
||||
object_info.size = 36;
|
||||
object_info.actual_size = 4;
|
||||
object_info.mod_time = Some(OffsetDateTime::from_unix_timestamp(written).expect("valid timestamp"));
|
||||
object_info.user_defined = HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())]).into();
|
||||
(roi, object_info)
|
||||
}
|
||||
|
||||
fn replica_written_secs() -> i64 {
|
||||
OffsetDateTime::parse(REPLICA_WRITTEN, &Rfc3339)
|
||||
.expect("valid replica time")
|
||||
.unix_timestamp()
|
||||
}
|
||||
|
||||
fn computed_etag_versions_response(versions: &[(&str, i64)]) -> String {
|
||||
let versions = versions
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, (version_id, size))| {
|
||||
format!(
|
||||
"<Version><Key>object</Key><VersionId>{version_id}</VersionId><IsLatest>{}</IsLatest><LastModified>{REPLICA_WRITTEN}</LastModified><ETag>"{COMPUTED_REPLICA_ETAG}"</ETag><Size>{size}</Size><StorageClass>STANDARD</StorageClass></Version>",
|
||||
index == 0
|
||||
)
|
||||
})
|
||||
.collect::<String>();
|
||||
xml_response(
|
||||
"200 OK",
|
||||
format!(
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><ListVersionsResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Name>target-bucket</Name><Prefix>object</Prefix><MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated>{versions}</ListVersionsResult>"
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
fn computed_etag_head_response(version_id: &str, size: i64) -> String {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nETag: \"{COMPUTED_REPLICA_ETAG}\"\r\nContent-Length: {size}\r\nx-amz-version-id: {version_id}\r\nLast-Modified: {REPLICA_WRITTEN_HTTP}\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// A Wasabi/AWS-shaped target that computes its own ETags: 404 on the
|
||||
/// source uuid, the listed versions, and a HEAD for any listed id.
|
||||
fn spawn_computed_etag_target(
|
||||
requests: usize,
|
||||
versions: &'static [(&'static str, i64)],
|
||||
) -> (String, std::thread::JoinHandle<Vec<String>>) {
|
||||
spawn_scripted_target_server(requests, move |line| {
|
||||
if line.starts_with("GET ") && line.contains("versions") {
|
||||
return computed_etag_versions_response(versions);
|
||||
}
|
||||
if line.starts_with("HEAD ")
|
||||
&& let Some((version_id, size)) = versions
|
||||
.iter()
|
||||
.find(|(version_id, _)| line.contains(&format!("versionId={version_id}")))
|
||||
{
|
||||
return computed_etag_head_response(version_id, *size);
|
||||
}
|
||||
if line.starts_with("HEAD ") {
|
||||
return empty_response("404 Not Found");
|
||||
}
|
||||
empty_response("500 Unexpected")
|
||||
})
|
||||
}
|
||||
|
||||
async fn resolve_heal_action(
|
||||
target: &Arc<TargetClient>,
|
||||
roi: &ReplicateObjectInfo,
|
||||
object_info: ObjectInfo,
|
||||
rinfo: &mut ReplicatedTargetInfo,
|
||||
) -> Option<(ReplicationAction, ObjectInfo)> {
|
||||
resolve_replicate_all_action(
|
||||
ReplicateAllActionContext {
|
||||
siblings: &NoSiblingLedger,
|
||||
roi,
|
||||
tgt_client: target,
|
||||
bucket: &roi.bucket,
|
||||
object: &roi.name,
|
||||
start_time: OffsetDateTime::now_utc(),
|
||||
ssec_audit_required: false,
|
||||
},
|
||||
object_info,
|
||||
rinfo,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_redrive_locates_encrypted_replica_by_shape_on_target_that_computes_own_etags() {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(3, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
|
||||
assert!(
|
||||
matches!(action, Some((ReplicationAction::None, _))),
|
||||
"an encrypted replica located by its shape must not be re-driven: {action:?}"
|
||||
);
|
||||
assert!(rinfo.error.is_none(), "{:?}", rinfo.error);
|
||||
assert_eq!(rinfo.target_version_id.as_deref(), Some(DRIFTED_ASSIGNED_VERSION_ID));
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 3, "HEAD by source id, ListObjectVersions, HEAD by assigned id: {seen:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_redrive_does_not_adopt_an_encrypted_replica_of_another_size_or_an_older_write() {
|
||||
// Size differs from the 4 plaintext bytes.
|
||||
static OTHER_SIZE: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 36)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, OTHER_SIZE);
|
||||
let target = test_target_client(endpoint);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
assert!(matches!(action, Some((ReplicationAction::All, _))), "{action:?}");
|
||||
server.join().expect("test HTTP server should finish");
|
||||
|
||||
// Same size, but written before the source version existed.
|
||||
static SAME_SIZE: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, SAME_SIZE);
|
||||
let target = test_target_client(endpoint);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() + 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
assert!(matches!(action, Some((ReplicationAction::All, _))), "{action:?}");
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_redrive_does_not_pick_among_several_encrypted_shape_matches() {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4), ("another-minted-id", 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
|
||||
assert!(
|
||||
matches!(action, Some((ReplicationAction::All, _))),
|
||||
"two same-shape versions are not evidence of either: {action:?}"
|
||||
);
|
||||
assert!(rinfo.target_version_id.is_none(), "{:?}", rinfo.target_version_id);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 2, "HEAD by source id, ListObjectVersions, no HEAD of a guess: {seen:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_redrive_keeps_exact_etag_identity_for_plaintext_sources() {
|
||||
// A same-size replica whose ETag differs is not this plaintext
|
||||
// version's replica: the shape criterion is for encrypted sources only.
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, mut object_info) = drifted_roi_and_object();
|
||||
object_info.mod_time = Some(OffsetDateTime::from_unix_timestamp(replica_written_secs() - 1).expect("valid timestamp"));
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
|
||||
assert!(matches!(action, Some((ReplicationAction::All, _))), "{action:?}");
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_redrive_never_adopts_an_unversioned_null_version_by_shape() {
|
||||
// An unversioned target's `null` version holds whatever generation
|
||||
// was written last; a same-size newer generation must not be claimed.
|
||||
static VERSIONS: &[(&str, i64)] = &[("null", 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(2, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
|
||||
assert!(matches!(action, Some((ReplicationAction::All, _))), "{action:?}");
|
||||
assert!(rinfo.target_version_id.is_none(), "{:?}", rinfo.target_version_id);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 2, "HEAD by source id, ListObjectVersions, nothing adopted: {seen:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_lookup_keeps_the_current_version_fallback_when_listing_is_denied() {
|
||||
// A bodiless 403 on the version-addressed HEAD may be a permission
|
||||
// denial; the same credentials then refuse ListObjectVersions. The
|
||||
// object must still be re-driven, as before content identity existed.
|
||||
let (endpoint, server) = spawn_scripted_target_server(3, |line| {
|
||||
let listing = line.starts_with("GET ") && line.contains("versions");
|
||||
let version_addressed_head = line.starts_with("HEAD ") && line.contains("versionId=");
|
||||
if listing || version_addressed_head {
|
||||
empty_response("403 Forbidden")
|
||||
} else if line.starts_with("HEAD ") {
|
||||
computed_etag_head_response(DRIFTED_ASSIGNED_VERSION_ID, 4)
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
|
||||
assert!(
|
||||
matches!(action, Some((ReplicationAction::All, _))),
|
||||
"a denied listing must fall back to the re-PUT, not stall: {action:?}"
|
||||
);
|
||||
assert!(rinfo.error.is_none(), "{:?}", rinfo.error);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(
|
||||
seen.len(),
|
||||
3,
|
||||
"HEAD by source id, ListObjectVersions, HEAD of the current version: {seen:?}"
|
||||
);
|
||||
assert!(seen[2].starts_with("HEAD ") && !seen[2].contains("versionId="), "{}", seen[2]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_ledger_replica_converges_on_an_aws_style_target() {
|
||||
// AWS rejects the RustFS uuid as malformed (400). The ledger names the
|
||||
// replica; its computed ETag cannot confirm it, its shape does.
|
||||
let (endpoint, server) = spawn_scripted_target_server(2, |line| {
|
||||
if line.starts_with("HEAD ") && line.contains(&format!("versionId={DRIFTED_ASSIGNED_VERSION_ID}")) {
|
||||
computed_etag_head_response(DRIFTED_ASSIGNED_VERSION_ID, 4)
|
||||
} else if line.starts_with("HEAD ") {
|
||||
empty_response("400 Bad Request")
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
let (roi, mut object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut user_defined = (*object_info.user_defined).clone();
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
&format!("{SUFFIX_REPLICATION_TARGET_VERSION_ARN_PREFIX}{}", target.arn),
|
||||
DRIFTED_ASSIGNED_VERSION_ID.to_string(),
|
||||
);
|
||||
object_info.user_defined = Arc::new(user_defined);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
let action = resolve_heal_action(&target, &roi, object_info, &mut rinfo).await;
|
||||
|
||||
assert!(matches!(action, Some((ReplicationAction::None, _))), "{action:?}");
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 2, "HEAD by source id, HEAD by the ledger id: {seen:?}");
|
||||
assert!(seen[1].contains(DRIFTED_ASSIGNED_VERSION_ID), "{}", seen[1]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resync_verification_counts_encrypted_replica_by_shape() {
|
||||
static VERSIONS: &[(&str, i64)] = &[(DRIFTED_ASSIGNED_VERSION_ID, 4)];
|
||||
let (endpoint, server) = spawn_computed_etag_target(3, VERSIONS);
|
||||
let target = test_target_client(endpoint);
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let (roi, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let mut st = TargetReplicationResyncStatus::default();
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (_, err) = verify_resync_head_result(head_result, &roi, Some(&object_info), &mut st, &target).await;
|
||||
|
||||
assert!(err.is_none(), "{err:?}");
|
||||
assert_eq!((st.replicated_count, st.failed_count), (1, 0));
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
fn encrypted_purge_source() -> Arc<VersionPurgeSource> {
|
||||
let (_, object_info) = encrypted_roi_and_object(replica_written_secs() - 1);
|
||||
let Some(ReplicaIdentity::Encrypted { shape, .. }) = replica_identity(&object_info) else {
|
||||
panic!("an SSE-S3 source must be identified by its shape");
|
||||
};
|
||||
Arc::new(VersionPurgeSource {
|
||||
etag: object_info.etag,
|
||||
encrypted_shape: Some(shape),
|
||||
ledger: HashMap::new(),
|
||||
ledger_corrupt: false,
|
||||
claimed_by_siblings: HashMap::new(),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_version_purge_without_ledger_deletes_the_unique_shape_match() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(2, |line| {
|
||||
if line.starts_with("GET ") && line.contains("versions") {
|
||||
computed_etag_versions_response(&[(DRIFTED_ASSIGNED_VERSION_ID, 4)])
|
||||
} else if line.starts_with("DELETE ") && line.contains(&format!("versionId={DRIFTED_ASSIGNED_VERSION_ID}")) {
|
||||
empty_response("204 No Content")
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(encrypted_purge_source())).await;
|
||||
|
||||
assert_eq!(rinfo.version_purge_status, VersionPurgeStatusType::Complete, "{:?}", rinfo.error);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 2, "ListObjectVersions, then DELETE by the located id: {seen:?}");
|
||||
assert!(
|
||||
seen[1].starts_with("DELETE ") && seen[1].contains(DRIFTED_ASSIGNED_VERSION_ID),
|
||||
"{}",
|
||||
seen[1]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_version_purge_refuses_several_shape_matches() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |line| {
|
||||
if line.starts_with("GET ") && line.contains("versions") {
|
||||
computed_etag_versions_response(&[(DRIFTED_ASSIGNED_VERSION_ID, 4), ("another-minted-id", 4)])
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), Some(encrypted_purge_source())).await;
|
||||
|
||||
assert_eq!(rinfo.version_purge_status, VersionPurgeStatusType::Failed);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "no DELETE of a guessed replica: {seen:?}");
|
||||
}
|
||||
|
||||
fn delete_marker_roi() -> ReplicateObjectInfo {
|
||||
ReplicateObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
@@ -6827,7 +7367,7 @@ mod tests {
|
||||
None,
|
||||
"the fixture must reproduce the codeless 405 the SDK yields for a bodiless HEAD"
|
||||
);
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, None, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_none(), "{err:?}");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 1, 0));
|
||||
@@ -6854,7 +7394,7 @@ mod tests {
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, None, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_some(), "a 405 must not count as a completed purge ({status:?})");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 0, 1), "{status:?}");
|
||||
@@ -6872,7 +7412,7 @@ mod tests {
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, None, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_none(), "{err:?}");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 1, 0));
|
||||
@@ -6889,7 +7429,7 @@ mod tests {
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, None, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_some(), "a 503 must not count as a propagated delete marker");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 0, 1));
|
||||
@@ -6915,7 +7455,7 @@ mod tests {
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, None, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_none(), "{err:?}");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 1, 0));
|
||||
@@ -6940,7 +7480,7 @@ mod tests {
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, None, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_some(), "a live latest version must not verify the marker");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 0, 1));
|
||||
@@ -6958,7 +7498,7 @@ mod tests {
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, None, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_some(), "a 403 must not count as a propagated delete marker");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 0, 1));
|
||||
@@ -7075,6 +7615,7 @@ mod tests {
|
||||
fn purge_source(ledger: &[(&str, &str)]) -> Arc<VersionPurgeSource> {
|
||||
Arc::new(VersionPurgeSource {
|
||||
etag: Some(DRIFTED_ETAG.to_string()),
|
||||
encrypted_shape: None,
|
||||
ledger: ledger
|
||||
.iter()
|
||||
.map(|(arn, version_id)| (arn.to_string(), version_id.to_string()))
|
||||
@@ -7279,6 +7820,7 @@ mod tests {
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
let source = Arc::new(VersionPurgeSource {
|
||||
etag: Some(DRIFTED_ETAG.to_string()),
|
||||
encrypted_shape: None,
|
||||
ledger: HashMap::new(),
|
||||
ledger_corrupt: false,
|
||||
claimed_by_siblings: HashMap::from([(target.arn.clone(), HashSet::from([DRIFTED_ASSIGNED_VERSION_ID.to_string()]))]),
|
||||
@@ -7366,6 +7908,7 @@ mod tests {
|
||||
let dobj = version_purge_dobj(&target.arn);
|
||||
let source = Arc::new(VersionPurgeSource {
|
||||
etag: Some(DRIFTED_ETAG.to_string()),
|
||||
encrypted_shape: None,
|
||||
ledger: HashMap::new(),
|
||||
ledger_corrupt: true,
|
||||
claimed_by_siblings: HashMap::new(),
|
||||
|
||||
@@ -37,8 +37,9 @@ use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemotePutObjectResponse, RemoveObjectOptions,
|
||||
ReplicaLocation, S3ClientError, TargetClient, resolve_read_api_version_id,
|
||||
AdvancedPutOptions, EncryptedReplicaShape, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
|
||||
RemotePutObjectResponse, RemoveObjectOptions, ReplicaCandidates, ReplicaEvidence, ReplicaIdentity, ReplicaLocation,
|
||||
S3ClientError, TargetClient, resolve_read_api_version_id,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
@@ -157,6 +158,19 @@ pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String
|
||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||
}
|
||||
|
||||
/// Whether the stored ETag of a replication source can be compared with the
|
||||
/// ETag a target returns for the bytes it received. Only an unencrypted source
|
||||
/// stores the MD5 of the bytes that go on the wire. A managed-SSE (SSE-S3 /
|
||||
/// SSE-KMS) or SSE-C source stores a keyed fingerprint (new writes) or a legacy
|
||||
/// plaintext MD5, and the target re-encrypts what it receives, so neither
|
||||
/// equality nor inequality of the two ETags proves anything about the replica.
|
||||
pub(crate) fn replication_source_etag_is_comparable(user_defined: &HashMap<String, String>) -> bool {
|
||||
matches!(
|
||||
classify_replication_source_encryption(user_defined),
|
||||
ReplicationSourceEncryption::Plaintext
|
||||
)
|
||||
}
|
||||
|
||||
/// HeadObjectOutput adapter over the pure SSE-C passthrough evidence
|
||||
/// judgment owned by `rustfs-replication`: extract the echoed
|
||||
/// customer-algorithm header and let the crate-owned policy decide.
|
||||
@@ -578,6 +592,22 @@ mod tests {
|
||||
use time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn replication_source_etag_is_comparable_only_for_plaintext_sources() {
|
||||
assert!(replication_source_etag_is_comparable(&HashMap::new()));
|
||||
for (key, value) in [
|
||||
(metadata_keys::SERVER_SIDE_ENCRYPTION, "AES256"),
|
||||
(metadata_keys::SERVER_SIDE_ENCRYPTION, "aws:kms"),
|
||||
(SSEC_ALGORITHM_HEADER, "AES256"),
|
||||
] {
|
||||
let metadata = HashMap::from([(key.to_string(), value.to_string())]);
|
||||
assert!(
|
||||
!replication_source_etag_is_comparable(&metadata),
|
||||
"{key}={value} must not expose its stored ETag to the replica audit"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// backlog#1735 A3b SSE key-case finding. Every RustFS writer persists the
|
||||
/// SSE intent as lowercase `x-amz-server-side-encryption`
|
||||
/// (`encryption_material_to_metadata`; header-derived keys come from a
|
||||
|
||||
@@ -356,7 +356,9 @@ fn apply_peer_replay_response(
|
||||
fn rename_data_grpc_stage(path: &str) -> bool {
|
||||
matches!(
|
||||
path,
|
||||
"/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation"
|
||||
"/node_service.NodeService/RenameData"
|
||||
| "/node_service.NodeService/RenameDataAtIncarnation"
|
||||
| "/node_service.NodeService/RenameDataWithQuotaFence"
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -26,8 +26,9 @@ use crate::disk::{
|
||||
BatchReadVersionReq, BatchReadVersionResp, CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection,
|
||||
DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
RenameDataResp, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
DiskOption, FileInfoVersions, FileReader, FileWriter, GuardedDeleteVersionsPayload, GuardedFileInfoVersions,
|
||||
PartTransactionAction, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, SnapshotLeaseToken,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, batch_read_version_one_by_one,
|
||||
disk_store::{
|
||||
DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING, ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, SKIP_IF_SUCCESS_BEFORE,
|
||||
get_drive_active_check_interval, get_drive_active_check_timeout, get_drive_disk_info_timeout, get_drive_list_dir_timeout,
|
||||
@@ -59,12 +60,12 @@ use rustfs_protos::proto_gen::node_service::RenamePartRequest;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest,
|
||||
ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RecoverDeleteTransactionRequest, RenameDataRequest, RenameFileRequest, SettlePartTransactionRequest,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest,
|
||||
UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -217,6 +218,24 @@ fn attach_mutation_body_digest<T>(
|
||||
crate::cluster::rpc::set_tonic_rolling_canonical_body_digest(request, &canonical_body).map_err(Error::other)
|
||||
}
|
||||
|
||||
fn attach_quota_mutation_body_digest<T>(
|
||||
request: &mut Request<T>,
|
||||
canonical_body: std::result::Result<Vec<u8>, std::num::TryFromIntError>,
|
||||
) -> Result<()> {
|
||||
let topology = crate::services::notification_sys::configured_fleet_topology_fingerprint()
|
||||
.ok_or_else(|| Error::other("quota mutation topology is not initialized or conflicts with startup"))?;
|
||||
let body = canonical_body.map_err(|_| Error::other("quota request length cannot be represented"))?;
|
||||
let body = rustfs_protos::canonical_quota_mutation_request_body(topology, &body)
|
||||
.map_err(|_| Error::other("quota topology envelope length cannot be represented"))?;
|
||||
request.metadata_mut().insert(
|
||||
rustfs_protos::QUOTA_MUTATION_TOPOLOGY_HEADER,
|
||||
topology
|
||||
.parse()
|
||||
.map_err(|_| Error::other("quota topology cannot be encoded"))?,
|
||||
);
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(request, &body).map_err(Error::other)
|
||||
}
|
||||
|
||||
fn decode_volume_infos(volume_infos: Vec<String>) -> Result<Vec<VolumeInfo>> {
|
||||
volume_infos
|
||||
.into_iter()
|
||||
@@ -899,6 +918,63 @@ fn spawn_control_channel_prewarm(addr: String) {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
async fn guarded_delete_versions_rpc(
|
||||
&self,
|
||||
volume: &str,
|
||||
payload: Vec<u8>,
|
||||
probe_only: bool,
|
||||
) -> Result<DeleteVersionsResponse> {
|
||||
if payload.len() > rustfs_protos::DELETE_RECOVERY_MAX_PAYLOAD_BYTES {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(GuardedDeleteVersionsRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_owned(),
|
||||
payload: payload.into(),
|
||||
probe_only,
|
||||
});
|
||||
let body = rustfs_protos::canonical_guarded_delete_versions_request_body(request.get_ref())
|
||||
.map_err(|_| DiskError::FileCorrupt)?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
// The additive method is mandatory: an old peer cannot safely delete
|
||||
// a guarded request through any legacy batch method.
|
||||
let response = client
|
||||
.guarded_delete_versions(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::GuardedDeleteUnsupported
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.ok_or(DiskError::FileCorrupt)?.into());
|
||||
}
|
||||
if response.error.is_some() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
|
||||
match self
|
||||
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::InspectOrphanDataDir { path: path.to_owned() })
|
||||
.await?
|
||||
{
|
||||
DeleteRecoveryRpcReply::OrphanDataDir { plain } => Ok(plain),
|
||||
_ => Err(Error::other("unexpected orphan data directory inspection reply")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn recover_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2300,7 +2376,13 @@ impl RemoteDisk {
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_rename_data_request_body(request.get_ref());
|
||||
let incarnation_bound = !request.get_ref().bucket_incarnation_id.is_empty();
|
||||
if scanner_publication_lease_token.is_some() || incarnation_bound {
|
||||
let quota_fenced = rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
);
|
||||
if quota_fenced {
|
||||
attach_quota_mutation_body_digest(&mut request, canonical_body)?;
|
||||
} else if scanner_publication_lease_token.is_some() || incarnation_bound {
|
||||
let canonical_body =
|
||||
canonical_body.map_err(|_| Error::other("rename_data request length cannot be represented"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical_body).map_err(Error::other)?;
|
||||
@@ -2315,7 +2397,10 @@ impl RemoteDisk {
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if incarnation_bound {
|
||||
if quota_fenced {
|
||||
// The method itself proves quota-token enforcement; an old peer must fail closed.
|
||||
client.rename_data_with_quota_fence(request).await
|
||||
} else if incarnation_bound {
|
||||
// Older peers return Unimplemented before mutation; never downgrade.
|
||||
client.rename_data_at_incarnation(request).await
|
||||
} else {
|
||||
@@ -2448,6 +2533,39 @@ impl RemoteDisk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for RemoteDisk {
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
|
||||
if !response.errors.is_empty() || !response.item_errors.is_empty() {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn delete_versions_guarded_recoverable(
|
||||
&self,
|
||||
volume: &str,
|
||||
versions: Vec<GuardedFileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
) -> Vec<Option<DiskError>> {
|
||||
let count = versions.len();
|
||||
let payload = GuardedDeleteVersionsPayload { versions, opts };
|
||||
if let Err(error) = payload.validate() {
|
||||
return vec![Some(error); count];
|
||||
}
|
||||
let encoded = match rmp_serde::to_vec_named(&payload) {
|
||||
Ok(encoded) => encoded,
|
||||
Err(error) => return vec![Some(error.into()); count],
|
||||
};
|
||||
match self.guarded_delete_versions_rpc(volume, encoded, false).await {
|
||||
Ok(response) => {
|
||||
// New guarded peers must return typed errors, never lossy strings.
|
||||
if response.item_errors.len() != count {
|
||||
return vec![Some(DiskError::FileCorrupt); count];
|
||||
}
|
||||
decode_delete_versions_errors(response, count)
|
||||
}
|
||||
Err(error) => vec![Some(error); count],
|
||||
}
|
||||
}
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
fn to_string(&self) -> String {
|
||||
self.endpoint.to_string()
|
||||
@@ -2955,8 +3073,15 @@ impl DiskAPI for RemoteDisk {
|
||||
.map_err(|_| Error::other("snapshot lease TTL cannot be represented"))?,
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "acquire_snapshot_lease")?;
|
||||
let response = client.acquire_snapshot_lease(request).await?.into_inner();
|
||||
let quota_fence = volume == crate::disk::RUSTFS_META_BUCKET && crate::disk::is_quota_mutation_fence_path(path);
|
||||
let response = if quota_fence {
|
||||
attach_quota_mutation_body_digest(&mut request, canonical_body)?;
|
||||
client.acquire_quota_mutation_fence(request).await?
|
||||
} else {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "acquire_snapshot_lease")?;
|
||||
client.acquire_snapshot_lease(request).await?
|
||||
}
|
||||
.into_inner();
|
||||
snapshot_lease_token_from_response(response)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
@@ -2997,8 +3122,15 @@ impl DiskAPI for RemoteDisk {
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let canonical_body = rustfs_protos::canonical_snapshot_lease_release_request_body(request.get_ref());
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "release_snapshot_lease")?;
|
||||
let response = client.release_snapshot_lease(request).await?.into_inner();
|
||||
let quota_fence = volume == crate::disk::RUSTFS_META_BUCKET && crate::disk::is_quota_mutation_fence_path(path);
|
||||
let response = if quota_fence {
|
||||
attach_quota_mutation_body_digest(&mut request, canonical_body)?;
|
||||
client.release_quota_mutation_fence(request).await?
|
||||
} else {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "release_snapshot_lease")?;
|
||||
client.release_snapshot_lease(request).await?
|
||||
}
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
@@ -3193,7 +3325,7 @@ impl DiskAPI for RemoteDisk {
|
||||
opts_bin: opts_bin.into(),
|
||||
});
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request();
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request(opts);
|
||||
if let Some(request_payload_bytes) = request_payload_bytes {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_sent_bytes(request_payload_bytes);
|
||||
}
|
||||
@@ -3271,6 +3403,7 @@ impl DiskAPI for RemoteDisk {
|
||||
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, encode_started);
|
||||
let request_payload_bytes = batch_read_version_attribution_enabled
|
||||
.then(|| batch_read_version_request_payload_len(&req, &batch_read_version_req, &batch_read_version_req_bin));
|
||||
let request_opts = req.opts;
|
||||
let batch_result = self
|
||||
.execute_with_timeout_for_op(
|
||||
"batch_read_version",
|
||||
@@ -3284,7 +3417,7 @@ impl DiskAPI for RemoteDisk {
|
||||
batch_read_version_req_bin: batch_read_version_req_bin.into(),
|
||||
});
|
||||
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request();
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_request(&request_opts);
|
||||
if let Some(request_payload_bytes) = request_payload_bytes {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_sent_bytes(
|
||||
request_payload_bytes.saturating_add(disk_len),
|
||||
@@ -3823,6 +3956,21 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn has_delete_transaction_evidence(&self, volume: &str, object: &str) -> Result<bool> {
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::Exists {
|
||||
object: object.to_owned(),
|
||||
},
|
||||
)
|
||||
.await?
|
||||
{
|
||||
DeleteRecoveryRpcReply::Evidence { present } => Ok(present),
|
||||
_ => Err(Error::other("unexpected delete recovery existence reply")),
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4322,6 +4470,101 @@ mod tests {
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
|
||||
let _without_proof = crate::services::notification_sys::without_live_cross_pool_fence_fleet_proof_for_test();
|
||||
assert!(crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_none());
|
||||
assert!(crate::services::notification_sys::configured_fleet_topology_fingerprint().is_some());
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let fence_path = crate::disk::quota_mutation_fence_path("bucket", "object");
|
||||
assert!(
|
||||
remote
|
||||
.acquire_snapshot_lease(crate::disk::RUSTFS_META_BUCKET, &fence_path)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
remote
|
||||
.release_snapshot_lease(crate::disk::RUSTFS_META_BUCKET, &fence_path, SnapshotLeaseToken::revoke_all())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut file_info = sample_rename_data_file_info();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut file_info.metadata,
|
||||
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
Uuid::new_v4().to_string(),
|
||||
);
|
||||
assert!(
|
||||
remote
|
||||
.rename_data("tmp", "source", file_info, "bucket", "object")
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let methods = peer
|
||||
.peer
|
||||
.unsupported_methods
|
||||
.lock()
|
||||
.expect("unsupported method list lock")
|
||||
.clone();
|
||||
assert_eq!(
|
||||
methods,
|
||||
vec![
|
||||
"/node_service.NodeService/AcquireQuotaMutationFence",
|
||||
"/node_service.NodeService/ReleaseQuotaMutationFence",
|
||||
"/node_service.NodeService/RenameDataWithQuotaFence",
|
||||
],
|
||||
"quota operations must never fall back to snapshot or unfenced rename methods"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guarded_delete_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
assert_eq!(remote.supports_guarded_delete_versions().await, Err(DiskError::GuardedDeleteUnsupported));
|
||||
let group = GuardedFileInfoVersions {
|
||||
group: FileInfoVersions {
|
||||
name: "object".into(),
|
||||
versions: vec![FileInfo::default()],
|
||||
..Default::default()
|
||||
},
|
||||
expected_current: crate::disk::CurrentDeleteGeneration::Absent,
|
||||
};
|
||||
let errors = remote
|
||||
.delete_versions_guarded_recoverable(
|
||||
"bucket",
|
||||
vec![group],
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(errors, vec![Some(DiskError::GuardedDeleteUnsupported)]);
|
||||
let methods = peer
|
||||
.peer
|
||||
.unsupported_methods
|
||||
.lock()
|
||||
.expect("unsupported method list lock")
|
||||
.clone();
|
||||
assert_eq!(
|
||||
methods,
|
||||
vec!["/node_service.NodeService/GuardedDeleteVersions"; 2],
|
||||
"neither capability failure nor guarded application may call a legacy deletion method"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recoverable_batch_delete_rejects_invalid_error_codes_and_length() {
|
||||
let errors = decode_delete_recovery_batch_errors(vec![None, Some(DiskError::FileNotFound.to_u32())], 2)
|
||||
@@ -4710,6 +4953,7 @@ mod tests {
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -4728,6 +4972,7 @@ mod tests {
|
||||
format_data,
|
||||
read_all_data,
|
||||
next_format_read_gate: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4874,13 +5119,20 @@ mod tests {
|
||||
Ok(grpc.unary(method, request).await)
|
||||
})
|
||||
}
|
||||
_ => Box::pin(async move {
|
||||
let mut response = http::Response::new(tonic::body::Body::default());
|
||||
let headers = response.headers_mut();
|
||||
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
|
||||
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
|
||||
Ok(response)
|
||||
}),
|
||||
_ => {
|
||||
self.peer
|
||||
.unsupported_methods
|
||||
.lock()
|
||||
.expect("unsupported method list lock")
|
||||
.push(request.uri().path().to_owned());
|
||||
Box::pin(async move {
|
||||
let mut response = http::Response::new(tonic::body::Body::default());
|
||||
let headers = response.headers_mut();
|
||||
headers.insert(tonic::Status::GRPC_STATUS, (tonic::Code::Unimplemented as i32).into());
|
||||
headers.insert(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE);
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -574,17 +574,20 @@ where
|
||||
>,
|
||||
{
|
||||
let h = HeaderMap::new();
|
||||
let mut rd = api
|
||||
.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
warn!("read_config_with_metadata: err: {:?}, file: {}", err, file);
|
||||
err
|
||||
}
|
||||
})?;
|
||||
let read = api.get_object_reader(RUSTFS_META_BUCKET, file, None, h, opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ConfigRead,
|
||||
read,
|
||||
);
|
||||
let mut rd = read.await.map_err(|err| {
|
||||
if err == Error::FileNotFound || matches!(err, Error::ObjectNotFound(_, _)) {
|
||||
Error::ConfigNotFound
|
||||
} else {
|
||||
warn!("read_config_with_metadata: err: {:?}, file: {}", err, file);
|
||||
err
|
||||
}
|
||||
})?;
|
||||
|
||||
let data = if let Some(max_bytes) = max_bytes {
|
||||
let object_size = usize::try_from(rd.object_info.size).map_err(|_| Error::CorruptedFormat)?;
|
||||
@@ -807,7 +810,18 @@ where
|
||||
>,
|
||||
{
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
match api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts).await {
|
||||
let write = api.put_object(RUSTFS_META_BUCKET, file, &mut put_data, opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let write = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ConfigWrite,
|
||||
async {
|
||||
crate::cluster::rpc::runtime_sources::observe_read_version_context_boundary(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::ConfigWrite,
|
||||
);
|
||||
write.await
|
||||
},
|
||||
);
|
||||
match write.await {
|
||||
Ok(object_info) => Ok(object_info),
|
||||
Err(err) => {
|
||||
if log_error {
|
||||
@@ -2839,7 +2853,7 @@ mod tests {
|
||||
use rustfs_config::{
|
||||
DEFAULT_DELIMITER, ENABLE_KEY, EnableState, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, MYSQL_DSN_STRING,
|
||||
MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_TABLE, SCANNER_CYCLE, SCANNER_IDLE_MODE, SCANNER_KEYS, SCANNER_SPEED,
|
||||
SCANNER_SUB_SYS,
|
||||
SCANNER_SUB_SYS, SCANNER_WALKERS_PER_DISK,
|
||||
};
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2869,6 +2883,11 @@ mod tests {
|
||||
#[derive(Debug, Default)]
|
||||
struct MetadataWriteStore {
|
||||
error: Option<Error>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
observations:
|
||||
std::sync::Mutex<Vec<(Option<rustfs_io_metrics::internode_metrics::ReadVersionEntrySource>, Option<&'static str>)>>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pending: bool,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -2899,7 +2918,20 @@ mod tests {
|
||||
_data: &mut Self::PutObjectReader,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> core::result::Result<Self::ObjectInfo, Self::Error> {
|
||||
Err(self.error.clone().expect("test store error should be configured"))
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
{
|
||||
self.observations.lock().expect("observations lock").push((
|
||||
crate::cluster::rpc::runtime_sources::current_read_version_entry_source(),
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
));
|
||||
if self.pending {
|
||||
std::future::pending::<()>().await;
|
||||
}
|
||||
}
|
||||
match self.error.clone() {
|
||||
Some(error) => Err(error),
|
||||
None => Ok(ObjectInfo::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2907,6 +2939,10 @@ mod tests {
|
||||
async fn save_config_preserves_retryable_system_volume_errors() {
|
||||
let store = Arc::new(MetadataWriteStore {
|
||||
error: Some(Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
observations: Default::default(),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pending: false,
|
||||
});
|
||||
let error =
|
||||
save_config_with_opts_inner(store, "buckets/example/.metadata.bin", Vec::new(), &ObjectOptions::default(), false)
|
||||
@@ -2918,6 +2954,59 @@ mod tests {
|
||||
Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn diagnostic_config_write_retains_origin_and_api_for_results_and_cancellation() {
|
||||
use crate::cluster::rpc::runtime_sources::{current_read_version_entry_source, with_read_version_entry};
|
||||
use rustfs_io_metrics::internode_metrics::ReadVersionEntrySource;
|
||||
use rustfs_io_metrics::s3_http_metrics::{S3HttpRequestGuard, current_s3_http_operation_context};
|
||||
|
||||
let mut guard = S3HttpRequestGuard::new("PUT");
|
||||
let put = guard.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::PutObject);
|
||||
current_s3_http_operation_context().expect("captured PutObject")
|
||||
});
|
||||
for earlier in [None, Some(ReadVersionEntrySource::TierConfigReload)] {
|
||||
for api in [None, Some(put)] {
|
||||
for case in ["success", "error", "cancelled"] {
|
||||
let store = Arc::new(MetadataWriteStore {
|
||||
error: (case == "error").then_some(Error::VolumeNotFound),
|
||||
pending: case == "cancelled",
|
||||
..Default::default()
|
||||
});
|
||||
let opts = ObjectOptions::default();
|
||||
let write = async {
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(1),
|
||||
save_config_with_opts_inner(store.clone(), "buckets/diagnostic/state", Vec::new(), &opts, false),
|
||||
)
|
||||
.await;
|
||||
match case {
|
||||
"success" => assert!(result.expect("write completes").is_ok()),
|
||||
"error" => {
|
||||
assert!(matches!(result.expect("error completes"), Err(Error::InsufficientWriteQuorum(_, _))))
|
||||
}
|
||||
_ => assert!(result.is_err(), "pending write is cancelled"),
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::s3_http_metrics::with_isolated_s3_http_operation_context(api, async {
|
||||
match earlier {
|
||||
Some(source) => with_read_version_entry(source, write).await,
|
||||
None => write.await,
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
*store.observations.lock().expect("observations lock"),
|
||||
vec![(Some(earlier.unwrap_or(ReadVersionEntrySource::ConfigWrite)), api.map(|_| "s3:PutObject"))]
|
||||
);
|
||||
assert_eq!(current_read_version_entry_source(), None, "write scope restores: {case}");
|
||||
assert_eq!(current_s3_http_operation_context(), None, "API scope restores: {case}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
use rustfs_lock::client::LockClient;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
|
||||
@@ -4112,6 +4201,8 @@ mod tests {
|
||||
let value = match *key {
|
||||
SCANNER_SPEED => "fast",
|
||||
SCANNER_IDLE_MODE => "false",
|
||||
// "1" is this key's default, which the diff encoding omits.
|
||||
SCANNER_WALKERS_PER_DISK => "2",
|
||||
_ => "1",
|
||||
};
|
||||
scanner_kvs.insert((*key).to_string(), value.to_string());
|
||||
@@ -4133,6 +4224,8 @@ mod tests {
|
||||
let expected = match *key {
|
||||
SCANNER_SPEED => "fast",
|
||||
SCANNER_IDLE_MODE => "false",
|
||||
// "1" is this key's default, which the diff encoding omits.
|
||||
SCANNER_WALKERS_PER_DISK => "2",
|
||||
_ => "1",
|
||||
};
|
||||
assert_eq!(decoded_scanner.get(key), expected, "scanner key {key} should survive roundtrip");
|
||||
|
||||
@@ -17,11 +17,11 @@ use rustfs_config::{
|
||||
DEFAULT_SCANNER_BITROT_CYCLE_SECS, DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS, DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||
DEFAULT_SCANNER_CYCLE_MAX_DURATION_SECS, DEFAULT_SCANNER_CYCLE_MAX_OBJECTS, DEFAULT_SCANNER_IDLE_MODE,
|
||||
DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, DEFAULT_SCANNER_SPEED,
|
||||
DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, SCANNER_ALERT_EXCESS_FOLDERS, SCANNER_ALERT_EXCESS_VERSION_SIZE,
|
||||
SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
|
||||
SCANNER_MAX_CONCURRENT_DISK_SCANS, SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_START_DELAY,
|
||||
SCANNER_YIELD_EVERY_N_OBJECTS,
|
||||
DEFAULT_SCANNER_WALKERS_PER_DISK, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, SCANNER_ALERT_EXCESS_FOLDERS,
|
||||
SCANNER_ALERT_EXCESS_VERSION_SIZE, SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT,
|
||||
SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY,
|
||||
SCANNER_IDLE_MODE, SCANNER_MAX_CONCURRENT_DISK_SCANS, SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED,
|
||||
SCANNER_START_DELAY, SCANNER_WALKERS_PER_DISK, SCANNER_YIELD_EVERY_N_OBJECTS,
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
@@ -92,6 +92,11 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
value: DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_WALKERS_PER_DISK.to_owned(),
|
||||
value: DEFAULT_SCANNER_WALKERS_PER_DISK.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_YIELD_EVERY_N_OBJECTS.to_owned(),
|
||||
value: DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS.to_string(),
|
||||
|
||||
@@ -3691,6 +3691,55 @@ fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
|
||||
}
|
||||
}
|
||||
|
||||
fn rebalance_worker_read_lock_error(err: rustfs_lock::LockError, object: &str) -> Error {
|
||||
match err {
|
||||
rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
object: object.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
},
|
||||
other => Error::Lock(other),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only admission to an already durable rebalance run. Keeping this
|
||||
/// separate from the activation writer prevents admission from authorizing a
|
||||
/// pool or rebalance metadata write under shared namespace locks.
|
||||
pub(crate) struct PoolRebalanceWorkerFence {
|
||||
pool_meta_guard: rustfs_lock::NamespaceLockGuard,
|
||||
rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
}
|
||||
|
||||
impl PoolRebalanceWorkerFence {
|
||||
pub(crate) fn set_fleet_proof(
|
||||
&mut self,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
) {
|
||||
self.fleet_proof = fleet_proof;
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_held(&self) -> Result<()> {
|
||||
if self.pool_meta_guard.is_lock_lost()
|
||||
|| self.pool_meta_guard.is_released()
|
||||
|| self.rebalance_meta_guard.is_lock_lost()
|
||||
|| self.rebalance_meta_guard.is_released()
|
||||
{
|
||||
return Err(Error::other("worker admission lock lost before rebalance activation"));
|
||||
}
|
||||
if self
|
||||
.fleet_proof
|
||||
.as_ref()
|
||||
.is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
|
||||
{
|
||||
return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct PoolRebalanceActivationFence {
|
||||
pool_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
rebalance_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
@@ -3762,6 +3811,35 @@ tokio::task_local! {
|
||||
pub(crate) static REBALANCE_ACTIVATION_LOCK_ATTEMPT: Arc<tokio::sync::Notify>;
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_pool_rebalance_worker_locks<S>(pool: Arc<S>) -> Result<PoolRebalanceWorkerFence>
|
||||
where
|
||||
S: crate::storage_api_contracts::namespace::NamespaceLocking<
|
||||
Error = Error,
|
||||
NamespaceLock = rustfs_lock::NamespaceLockWrapper,
|
||||
>,
|
||||
{
|
||||
// Match activation and entry admission: rebalance.bin -> pool.bin. Joining
|
||||
// the same durable run must coexist with entries retaining their run fence.
|
||||
let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
|
||||
#[cfg(test)]
|
||||
let _ = REBALANCE_ACTIVATION_LOCK_ATTEMPT.try_with(|attempted| attempted.notify_one());
|
||||
let rebalance_meta_guard = rebalance_meta_lock
|
||||
.get_read_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| rebalance_worker_read_lock_error(err, REBAL_META_NAME))?;
|
||||
let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
|
||||
let pool_meta_guard = pool_meta_lock
|
||||
.get_read_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| rebalance_worker_read_lock_error(err, POOL_META_NAME))?;
|
||||
|
||||
Ok(PoolRebalanceWorkerFence {
|
||||
pool_meta_guard,
|
||||
rebalance_meta_guard,
|
||||
fleet_proof: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
|
||||
pool: Arc<S>,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
@@ -6685,12 +6763,12 @@ fn pool_meta_cas_options(object: &str, token: &PoolMetaCasToken, fence: &PoolMet
|
||||
fence.ensure_held()?;
|
||||
let mut opts = ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
no_lock: true,
|
||||
http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
|
||||
..Default::default()
|
||||
};
|
||||
fence.add_to_options(&mut opts, object);
|
||||
opts.write_completion = crate::object_api::WriteCompletion::TailDrainedWithOwnerRepair;
|
||||
Ok(opts)
|
||||
}
|
||||
|
||||
@@ -6735,7 +6813,11 @@ where
|
||||
"payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&data)),
|
||||
"if_match": opts.http_preconditions.as_ref().and_then(|p| p.if_match.as_deref()),
|
||||
"if_none_match": opts.http_preconditions.as_ref().and_then(|p| p.if_none_match.as_deref()),
|
||||
"tail_drained": opts.write_completion == crate::object_api::WriteCompletion::TailDrained,
|
||||
"tail_drained": matches!(
|
||||
opts.write_completion,
|
||||
crate::object_api::WriteCompletion::TailDrained
|
||||
| crate::object_api::WriteCompletion::TailDrainedWithOwnerRepair
|
||||
),
|
||||
"no_lock": opts.no_lock,
|
||||
})
|
||||
});
|
||||
@@ -12430,6 +12512,18 @@ impl ECStore {
|
||||
Ok(pool_meta)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_runtime_pool_meta_under_worker_fence(
|
||||
&self,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
worker_fence: &PoolRebalanceWorkerFence,
|
||||
operation: &str,
|
||||
) -> Result<PoolMeta> {
|
||||
worker_fence.ensure_held()?;
|
||||
let pool_meta = self.load_runtime_pool_meta_observing(write_state, operation).await?;
|
||||
worker_fence.ensure_held()?;
|
||||
Ok(pool_meta)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_runtime_pool_meta_under_activation_fence(
|
||||
&self,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
@@ -24692,7 +24786,7 @@ mod pools_tests {
|
||||
_ => panic!("phase fixture must receive valid pool metadata"),
|
||||
};
|
||||
assert!(opts.max_parity && opts.no_lock);
|
||||
assert_eq!(opts.write_completion, crate::object_api::WriteCompletion::TailDrained);
|
||||
assert_eq!(opts.write_completion, crate::object_api::WriteCompletion::TailDrainedWithOwnerRepair);
|
||||
self.trace.calls.lock().expect("trace lock").push(WriteCall {
|
||||
pool: self.pool,
|
||||
phase,
|
||||
|
||||
@@ -893,6 +893,8 @@ fn is_data_movement_rewritten_metadata(object_info: &ObjectInfo, key: &str, norm
|
||||
SUFFIX_ACTUAL_SIZE,
|
||||
SUFFIX_CRC,
|
||||
SUFFIX_PART_CHECKSUMS,
|
||||
// Inline placement is local to each physical copy, like its data directory.
|
||||
rustfs_utils::http::SUFFIX_INLINE_DATA,
|
||||
crate::object_api::ENCRYPTED_PART_LAYOUT_QUORUM_SUFFIX,
|
||||
]
|
||||
.iter()
|
||||
@@ -3532,6 +3534,23 @@ mod tests {
|
||||
assert!(overwrite_resume_for_target(&source, source.clone()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_movement_identity_accepts_inline_layout_change_only() {
|
||||
let source = overwrite_equivalence_source();
|
||||
let mut target = source.clone();
|
||||
rustfs_utils::http::insert_str(
|
||||
Arc::make_mut(&mut target.user_defined),
|
||||
rustfs_utils::http::SUFFIX_INLINE_DATA,
|
||||
"true".to_string(),
|
||||
);
|
||||
assert!(is_equivalent_data_movement_object(&source, &target));
|
||||
assert!(is_equivalent_data_movement_object(&target, &source));
|
||||
|
||||
Arc::make_mut(&mut target.user_defined).insert("x-amz-meta-inline-data".to_string(), "user value".to_string());
|
||||
assert!(!is_equivalent_data_movement_object(&source, &target));
|
||||
assert!(!is_equivalent_data_movement_object(&target, &source));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_data_movement_overwrite_resume_accepts_destination_marker() {
|
||||
let source = overwrite_equivalence_source();
|
||||
|
||||
@@ -110,19 +110,17 @@ async fn read_replica(set: Arc<SetDisks>) -> Result<ScannerPauseBacklogRetiremen
|
||||
set_index: set.set_index,
|
||||
data: None,
|
||||
};
|
||||
let reader = match set
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
SCANNER_PAUSE_BACKLOG_PATH,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let read = set.get_object_reader(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, None, HeaderMap::new(), &opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ScannerBacklogRead,
|
||||
read,
|
||||
);
|
||||
let reader = match read.await {
|
||||
Ok(reader) => reader,
|
||||
Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {
|
||||
return Ok(ScannerPauseBacklogRetirementRead { replica, etag: None });
|
||||
|
||||
@@ -1325,6 +1325,18 @@ fn select_admin_data_usage_snapshot(
|
||||
&& authoritative.is_complete_bucket_usage_snapshot()
|
||||
&& observed_data_usage_is_newer(&observed, &authoritative) =>
|
||||
{
|
||||
// A bucket can span several sets. Without a current or retained
|
||||
// generation for every live set, its observed sum is only a subset,
|
||||
// not a replacement for the last complete bucket total. Keep the
|
||||
// baseline's timestamp and provenance until coverage is available.
|
||||
if authoritative_format
|
||||
&& observed
|
||||
.usage_snapshot_set_states
|
||||
.iter()
|
||||
.any(|state| !state.tombstone && state.scanner_cycle.is_none())
|
||||
{
|
||||
return (authoritative, authoritative_format);
|
||||
}
|
||||
(merge_partial_observation_for_admin(authoritative, observed), true)
|
||||
}
|
||||
Some(observed) if observed_data_usage_is_newer(&observed, &authoritative) => (observed, true),
|
||||
@@ -2529,19 +2541,17 @@ where
|
||||
use crate::object_api::ObjectOptions;
|
||||
use http::HeaderMap;
|
||||
|
||||
match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
key,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
let read = store.get_object_reader(RUSTFS_META_BUCKET, key, None, HeaderMap::new(), &opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::DataUsageCacheRead,
|
||||
read,
|
||||
);
|
||||
match read.await {
|
||||
// A cache object that fails to decode is treated as absent rather than
|
||||
// as an error: a corrupt cache should not stall the caller, and the
|
||||
// next scanner pass rewrites it.
|
||||
@@ -3064,6 +3074,8 @@ mod tests {
|
||||
transient_failures: Mutex<usize>,
|
||||
reads: Mutex<Vec<String>>,
|
||||
terminal_error: Mutex<Option<Error>>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
record_context: bool,
|
||||
}
|
||||
|
||||
impl UsageCacheReadStore {
|
||||
@@ -3072,6 +3084,8 @@ mod tests {
|
||||
transient_failures: Mutex::new(n),
|
||||
reads: Mutex::new(Vec::new()),
|
||||
terminal_error: Mutex::new(None),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
record_context: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3080,6 +3094,8 @@ mod tests {
|
||||
transient_failures: Mutex::new(0),
|
||||
reads: Mutex::new(Vec::new()),
|
||||
terminal_error: Mutex::new(Some(error)),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
record_context: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3106,6 +3122,12 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
if self.record_context {
|
||||
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_request(
|
||||
&crate::disk::ReadOptions::default(),
|
||||
);
|
||||
}
|
||||
self.reads.lock().await.push(object.to_string());
|
||||
if let Some(error) = self.terminal_error.lock().await.clone() {
|
||||
return Err(error);
|
||||
@@ -3205,6 +3227,89 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_usage_cache_read_preserves_api_and_earlier_entry() {
|
||||
use crate::cluster::rpc::runtime_sources::with_read_version_entry;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_io_metrics::internode_metrics::ReadVersionEntrySource;
|
||||
use rustfs_io_metrics::s3_http_metrics::{
|
||||
S3HttpRequestGuard, current_s3_http_operation_context, with_s3_http_operation_context,
|
||||
};
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("diagnostic usage cache runtime");
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshots = recorder.snapshotter();
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let get = request.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
|
||||
current_s3_http_operation_context().expect("dispatched GET operation")
|
||||
});
|
||||
request.response(200);
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let ambient = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture();
|
||||
for (operation, source) in [
|
||||
(None, None),
|
||||
(Some(get), None),
|
||||
(None, Some(ReadVersionEntrySource::ServerInfoPools)),
|
||||
(Some(get), Some(ReadVersionEntrySource::StoreObjectReader)),
|
||||
] {
|
||||
let store = UsageCacheReadStore {
|
||||
record_context: true,
|
||||
..Default::default()
|
||||
};
|
||||
with_s3_http_operation_context(operation, async {
|
||||
let load = load_data_usage_cache(&store, "usage-cache");
|
||||
let cache = match source {
|
||||
Some(source) => with_read_version_entry(source, load).await,
|
||||
None => load.await,
|
||||
}
|
||||
.expect("missing diagnostic cache is harmless");
|
||||
assert!(cache.cache.is_empty());
|
||||
assert_eq!(store.read_keys().await.len(), 2, "both prefixed and legacy cache reads are observed");
|
||||
})
|
||||
.await;
|
||||
assert_eq!(
|
||||
crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture(),
|
||||
ambient,
|
||||
"cache reads must restore the caller's complete diagnostic scope"
|
||||
);
|
||||
}
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
});
|
||||
});
|
||||
let rows = snapshots.snapshot().into_vec();
|
||||
for (source, operation, state) in [
|
||||
("data_usage_cache_read", "unattributed", "absent"),
|
||||
("data_usage_cache_read", "s3:GetObject", "preserved"),
|
||||
("server_info_pools", "unattributed", "absent"),
|
||||
("store_object_reader", "s3:GetObject", "preserved"),
|
||||
] {
|
||||
assert!(
|
||||
rows.iter().any(|(key, _, _, value)| {
|
||||
let labels = key
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key(), label.value()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
key.key().name() == "rustfs_storage_diagnostic_context_transitions_total"
|
||||
&& labels.get("entry_source") == Some(&source)
|
||||
&& labels.get("boundary") == Some(&"remote_rpc")
|
||||
&& labels.get("parent_api_operation") == Some(&operation)
|
||||
&& labels.get("child_api_operation") == Some(&operation)
|
||||
&& labels.get("api_context") == Some(&state)
|
||||
&& matches!(value, DebugValue::Counter(2))
|
||||
}),
|
||||
"cache reads must retain API and the earliest observed entry: {source}/{operation}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_usage_removal_surfaces_missing_system_volume_as_read_quorum() {
|
||||
for cause in [Error::BucketNotFound(RUSTFS_META_BUCKET.to_string()), Error::VolumeNotFound] {
|
||||
@@ -3718,6 +3823,169 @@ mod tests {
|
||||
assert!(selected.unknown_tier_stats.is_none());
|
||||
}
|
||||
|
||||
fn usage_snapshots_with_missing_observed_set() -> (DataUsageInfo, DataUsageInfo) {
|
||||
let baseline_time = SystemTime::UNIX_EPOCH + Duration::from_secs(10);
|
||||
let mut authoritative = data_usage_info_for_test("large-bucket", 510_000, 580_000_000_000, baseline_time);
|
||||
authoritative.scanner_epoch = Some(4);
|
||||
authoritative.scanner_cycle = Some(10);
|
||||
authoritative.usage_snapshot_converged = Some(true);
|
||||
|
||||
let mut observed = data_usage_info_for_test("large-bucket", 3_000, 3_000_000_000, baseline_time + Duration::from_secs(1));
|
||||
observed.scanner_epoch = Some(4);
|
||||
observed.scanner_cycle = Some(11);
|
||||
observed.usage_snapshot_complete = false;
|
||||
observed.usage_snapshot_partial = true;
|
||||
observed.usage_snapshot_converged = Some(false);
|
||||
observed.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
|
||||
observed.usage_snapshot_set_states = vec![
|
||||
rustfs_data_usage::DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(11),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
..Default::default()
|
||||
},
|
||||
rustfs_data_usage::DataUsageSnapshotSetState {
|
||||
pool_index: 1,
|
||||
set_index: 0,
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
..Default::default()
|
||||
},
|
||||
];
|
||||
assert!(
|
||||
observed.is_valid_partial_snapshot(),
|
||||
"a missing set is valid diagnostic progress, not a bucket total"
|
||||
);
|
||||
(authoritative, observed)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_admin_observation_does_not_replace_totals_with_a_missing_set() {
|
||||
let (authoritative, observed) = usage_snapshots_with_missing_observed_set();
|
||||
let identity = authoritative.snapshot_identity();
|
||||
|
||||
let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(observed));
|
||||
|
||||
assert_eq!(selected.buckets_usage["large-bucket"].objects_count, 510_000);
|
||||
assert_eq!(selected.buckets_usage["large-bucket"].size, 580_000_000_000);
|
||||
assert_eq!(selected.objects_total_count, 510_000);
|
||||
assert_eq!(selected.objects_total_size, 580_000_000_000);
|
||||
assert_eq!(
|
||||
selected.snapshot_identity(),
|
||||
identity,
|
||||
"retained counts must not gain a newer timestamp or generation"
|
||||
);
|
||||
assert!(selected.is_complete_bucket_usage_snapshot());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_admin_observation_does_not_promote_a_legacy_baseline() {
|
||||
let (legacy, observed) = usage_snapshots_with_missing_observed_set();
|
||||
let (selected, current_format) = select_admin_data_usage_snapshot(legacy, false, Some(observed));
|
||||
let (normalized, _) = normalize_loaded_data_usage(selected, current_format).await;
|
||||
|
||||
assert!(normalized.is_valid_partial_snapshot());
|
||||
assert!(!normalized.is_complete_bucket_usage_snapshot());
|
||||
assert_eq!(normalized.usage_snapshot_converged, Some(false));
|
||||
assert_eq!(normalized.objects_total_count, 3_000);
|
||||
assert_eq!(normalized.objects_total_size, 3_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_admin_observation_accepts_decreases_when_every_set_is_covered() {
|
||||
let (authoritative, mut observed) = usage_snapshots_with_missing_observed_set();
|
||||
// A compatible retained generation is useful even when it is not a
|
||||
// completed scan of the current cycle. A tombstone explicitly removes
|
||||
// a set; neither case is the same as an unavailable live set.
|
||||
for tombstone in [false, true] {
|
||||
observed.usage_snapshot_set_states[1].tombstone = tombstone;
|
||||
observed.usage_snapshot_set_states[1].scanner_epoch = (!tombstone).then_some(4);
|
||||
observed.usage_snapshot_set_states[1].scanner_cycle = (!tombstone).then_some(10);
|
||||
let (selected, _) = select_admin_data_usage_snapshot(authoritative.clone(), true, Some(observed.clone()));
|
||||
|
||||
assert_eq!(selected.objects_total_count, 3_000);
|
||||
assert_eq!(selected.objects_total_size, 3_000_000_000);
|
||||
assert!(selected.is_valid_partial_snapshot());
|
||||
assert!(!selected.is_complete_bucket_usage_snapshot());
|
||||
assert_eq!(selected.usage_snapshot_converged, Some(false));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn partial_admin_observation_preserves_persisted_totals_across_cache_reload() {
|
||||
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
|
||||
use crate::config::com::save_config;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let (authoritative, observed) = usage_snapshots_with_missing_observed_set();
|
||||
let primary_bytes = serde_json::to_vec(&authoritative).expect("encode complete usage");
|
||||
let observed_bytes = serde_json::to_vec(&observed).expect("encode partial observation");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), primary_bytes.clone())
|
||||
.await
|
||||
.expect("persist complete usage");
|
||||
save_config(store.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), observed_bytes.clone())
|
||||
.await
|
||||
.expect("persist observation with a missing set");
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
|
||||
for reload in [false, false, true] {
|
||||
if reload {
|
||||
// Recreate process-local usage state, not an OS process restart.
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
let mut displayed = load_admin_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("load admin usage through the real snapshot cache");
|
||||
apply_bucket_usage_memory_overlay(&mut displayed).await;
|
||||
assert_eq!(displayed.objects_total_count, 510_000);
|
||||
assert_eq!(displayed.objects_total_size, 580_000_000_000);
|
||||
assert_eq!(displayed.bucket_sizes["large-bucket"], 580_000_000_000);
|
||||
assert_eq!(displayed.snapshot_identity(), authoritative.snapshot_identity());
|
||||
}
|
||||
let quota = load_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("load quota authority");
|
||||
assert_eq!(quota.objects_total_size, 580_000_000_000);
|
||||
assert_eq!(
|
||||
read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read primary"),
|
||||
primary_bytes
|
||||
);
|
||||
assert_eq!(
|
||||
read_config(store.clone(), DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read diagnostic observation"),
|
||||
observed_bytes,
|
||||
"admin reads must not rewrite either persisted snapshot"
|
||||
);
|
||||
|
||||
let mut reconciled = data_usage_info_for_test("large-bucket", 2, 42, SystemTime::UNIX_EPOCH + Duration::from_secs(12));
|
||||
reconciled.scanner_epoch = Some(4);
|
||||
reconciled.scanner_cycle = Some(12);
|
||||
reconciled.usage_snapshot_converged = Some(true);
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&reconciled).expect("encode confirmed deletion"),
|
||||
)
|
||||
.await
|
||||
.expect("persist a later complete scan after deletion");
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let displayed = load_admin_data_usage_from_backend_cached(store)
|
||||
.await
|
||||
.expect("load confirmed decrease");
|
||||
assert_eq!((displayed.objects_total_count, displayed.objects_total_size), (2, 42));
|
||||
assert_eq!(displayed.snapshot_identity(), reconciled.snapshot_identity());
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
|
||||
let store = UsageCasStore::default();
|
||||
|
||||
@@ -39,14 +39,14 @@ use rustfs_protos::{
|
||||
};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
time::Duration,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use tracing::{error, trace, warn};
|
||||
|
||||
const EVENT_SERVER_PING: &str = "server_ping";
|
||||
const EVENT_SERVER_INFO: &str = "server_info";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_DIAGNOSTICS: &str = "diagnostics";
|
||||
|
||||
@@ -276,16 +276,10 @@ fn collect_runtime_server_stats() -> (MemStats, u64, u64) {
|
||||
}
|
||||
|
||||
pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
let nowt: OffsetDateTime = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("get_server_info start {:?}", nowt);
|
||||
let started = tracing::enabled!(tracing::Level::TRACE).then(Instant::now);
|
||||
|
||||
let local = get_local_server_property().await;
|
||||
|
||||
let after1 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("get_local_server_property end {:?}", after1 - nowt);
|
||||
|
||||
let mut servers = {
|
||||
if let Some(sys) = runtime_sources::notification_sys() {
|
||||
sys.server_info().await
|
||||
@@ -294,9 +288,6 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
}
|
||||
};
|
||||
|
||||
let after2 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("server_info end {:?}", after2 - after1);
|
||||
servers.push(local);
|
||||
|
||||
let mut buckets = rustfs_madmin::Buckets::default();
|
||||
@@ -331,15 +322,8 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
);
|
||||
}
|
||||
|
||||
let after3 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("load_data_usage_from_backend end {:?}", after3 - after2);
|
||||
|
||||
let backend_info = StorageAdminApi::backend_info(store.as_ref()).await;
|
||||
|
||||
let after4 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("backend_info end {:?}", after4 - after3);
|
||||
if let Some(endpoints) = runtime_sources::endpoint_pools() {
|
||||
let (added, report) = reconcile_servers_with_endpoint_topology(&mut servers, &endpoints);
|
||||
if added > 0 || !report.is_complete() {
|
||||
@@ -361,9 +345,6 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
}
|
||||
let (online_disks, offline_disks, unknown_disks) = get_online_offline_disks_stats(&all_disks);
|
||||
|
||||
let after5 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("get_online_offline_disks_stats end {:?}", after5 - after4);
|
||||
backend = rustfs_madmin::ErasureBackend {
|
||||
backend_type: rustfs_madmin::BackendType::ErasureType,
|
||||
online_disks: online_disks.sum(),
|
||||
@@ -376,14 +357,24 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
};
|
||||
if get_pools {
|
||||
pools = get_pools_info(&all_disks).await.unwrap_or_default();
|
||||
let after6 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("get_pools_info end {:?}", after6 - after5);
|
||||
}
|
||||
}
|
||||
|
||||
let services = rustfs_madmin::Services::default();
|
||||
|
||||
if let Some(started) = started {
|
||||
trace!(
|
||||
event = EVENT_SERVER_INFO,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DIAGNOSTICS,
|
||||
state = "completed",
|
||||
mode,
|
||||
include_pools = get_pools,
|
||||
duration_ms = started.elapsed().as_secs_f64() * 1000.0,
|
||||
"Server info collected"
|
||||
);
|
||||
}
|
||||
|
||||
InfoMessage {
|
||||
mode: Some(mode.to_string()),
|
||||
domain: None,
|
||||
@@ -695,12 +686,16 @@ async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32,
|
||||
|
||||
if erasure_set.id == 0 {
|
||||
erasure_set.id = d.set_index;
|
||||
match load_data_usage_cache(
|
||||
let load = load_data_usage_cache(
|
||||
store.pools[d.pool_index as usize].disk_set[d.set_index as usize].as_ref(),
|
||||
DATA_USAGE_CACHE_NAME,
|
||||
)
|
||||
.await
|
||||
{
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let load = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ServerInfoPools,
|
||||
load,
|
||||
);
|
||||
match load.await {
|
||||
Ok(cache) => apply_erasure_set_usage(&cache, erasure_set),
|
||||
Err(_) => erasure_set.usage_error = Some(DATA_USAGE_UNAVAILABLE_ERROR.to_string()),
|
||||
}
|
||||
|
||||
@@ -19,8 +19,8 @@ use crate::disk::io_admission::{
|
||||
};
|
||||
use crate::disk::{
|
||||
CheckPartsResp, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint,
|
||||
Error, FileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, Result,
|
||||
SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
Error, FileInfoVersions, GuardedFileInfoVersions, MmapCopyStageMetrics, ReadMultipleReq, ReadMultipleResp, ReadOptions,
|
||||
RenameDataResp, Result, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions,
|
||||
health_state::{
|
||||
RuntimeDriveHealthState, classify_drive_recovery, get_drive_returning_probe_interval,
|
||||
get_drive_returning_success_threshold, get_drive_suspect_failure_threshold, record_drive_offline_duration,
|
||||
@@ -1829,21 +1829,7 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
// Clean up
|
||||
disk.delete(
|
||||
test_bucket,
|
||||
test_filename,
|
||||
DeleteOptions {
|
||||
recursive: false,
|
||||
immediate: false,
|
||||
undo_write: false,
|
||||
undo_delete: false,
|
||||
old_data_dir: None,
|
||||
expected_delete_marker: None,
|
||||
recoverable_delete: false,
|
||||
delete_recovery_bucket_incarnation: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
disk.delete(test_bucket, test_filename, DeleteOptions::default()).await?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
@@ -2078,6 +2064,58 @@ impl LocalDiskWrapper {
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_versions_guarded_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
versions: Vec<GuardedFileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Vec<Option<Error>> {
|
||||
let version_count = versions.len();
|
||||
self.metrics.record_operation_call("delete_versions");
|
||||
// Check if disk is faulty before proceeding
|
||||
if self.health.is_faulty() {
|
||||
self.metrics.record_availability_error();
|
||||
return vec![Some(DiskError::FaultyDisk); versions.len()];
|
||||
}
|
||||
|
||||
// Check if disk is stale
|
||||
if let Err(e) = self.check_disk_stale().await {
|
||||
self.metrics.record_availability_error();
|
||||
return vec![Some(e); versions.len()];
|
||||
}
|
||||
|
||||
let _io_permit = match self.acquire_disk_io_permit().await {
|
||||
Ok(permit) => permit,
|
||||
Err(error) => return vec![Some(error); version_count],
|
||||
};
|
||||
|
||||
// Record operation start
|
||||
self.health.last_started.store(current_unix_nanos(), Ordering::Relaxed);
|
||||
let waiting_guard = self.health.waiting_guard();
|
||||
let metric_waiting_guard = self.metrics.waiting_guard();
|
||||
let started = Instant::now();
|
||||
|
||||
// Execute the operation
|
||||
let result = self
|
||||
.disk
|
||||
.delete_versions_guarded_with_namespace_owner(volume, versions, opts, namespace_owner)
|
||||
.await;
|
||||
self.metrics.record_operation_latency("delete_versions", started.elapsed());
|
||||
self.record_batch_delete_error_metrics(&result);
|
||||
|
||||
drop(waiting_guard);
|
||||
drop(metric_waiting_guard);
|
||||
let has_err = result.iter().any(|e| e.is_some());
|
||||
if !has_err {
|
||||
// Log success and decrement waiting counter
|
||||
self.health.record_operation_success(&self.endpoint(), "operation_success");
|
||||
self.metrics.record_mutation_success(DiskMetricMutation::Delete);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub async fn get_current_disk_id(&self) -> Option<Uuid> {
|
||||
*self.disk_id.read().await
|
||||
}
|
||||
@@ -2321,6 +2359,21 @@ impl LocalDiskWrapper {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDiskWrapper {
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
// The local implementation shares this binary's capability. Availability
|
||||
// and disk identity remain mutation checks and write-quorum inputs.
|
||||
self.disk.supports_guarded_delete_versions().await
|
||||
}
|
||||
async fn delete_versions_guarded_recoverable(
|
||||
&self,
|
||||
volume: &str,
|
||||
versions: Vec<GuardedFileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
) -> Vec<Option<DiskError>> {
|
||||
self.delete_versions_guarded_with_namespace_owner(volume, versions, opts, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_version_recoverable(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2351,6 +2404,14 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
opts.recoverable_delete = true;
|
||||
self.delete_versions(volume, versions, opts).await
|
||||
}
|
||||
async fn has_delete_transaction_evidence(&self, volume: &str, object: &str) -> Result<bool> {
|
||||
self.track_disk_health_with_op(
|
||||
"has_delete_transaction_evidence",
|
||||
|| async { self.disk.has_delete_transaction_evidence(volume, object).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -2743,6 +2804,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
async fn read_version(
|
||||
&self,
|
||||
org_volume: &str,
|
||||
@@ -2751,7 +2813,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<FileInfo> {
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
let operation = self.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_version",
|
||||
|io_permit| async move {
|
||||
with_disk_io_permit(io_permit, self.disk.read_version(org_volume, volume, path, version_id, opts)).await
|
||||
@@ -2759,8 +2821,20 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
get_max_timeout_duration(),
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
{
|
||||
if crate::cluster::rpc::runtime_sources::current_read_version_request_caller().is_some() {
|
||||
operation.await
|
||||
} else {
|
||||
let caller = crate::cluster::rpc::runtime_sources::read_version_request_caller_from_location(
|
||||
std::panic::Location::caller().file(),
|
||||
);
|
||||
crate::cluster::rpc::runtime_sources::with_read_version_request_caller(caller, operation).await
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
operation.await
|
||||
}
|
||||
|
||||
async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result<RawFileInfo> {
|
||||
@@ -3921,6 +3995,97 @@ mod tests {
|
||||
assert_eq!(snapshot.total_errors_availability, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_guarded_delete_capability_is_independent_of_faulty_or_stale_mutation_rejection() {
|
||||
for stale in [false, true] {
|
||||
let dir = tempfile::tempdir().expect("create local disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 disk path")).expect("parse local endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("create local disk"));
|
||||
let volume = "guarded-capability";
|
||||
let object = "object";
|
||||
disk.make_volume(volume).await.expect("create object volume");
|
||||
let mut info = FileInfo::new(object, 1, 0);
|
||||
info.name = object.to_owned();
|
||||
info.version_id = Some(Uuid::nil());
|
||||
info.data_dir = Some(Uuid::new_v4());
|
||||
info.mod_time = Some(::time::OffsetDateTime::now_utc());
|
||||
info.erasure.index = 1;
|
||||
info.size = 8;
|
||||
info.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 8,
|
||||
actual_size: 8,
|
||||
..Default::default()
|
||||
}];
|
||||
info.data = Some(Bytes::from_static(b"original"));
|
||||
info.metadata.insert("etag".into(), "etag-a".into());
|
||||
info.set_inline_data();
|
||||
disk.write_metadata(volume, volume, object, info.clone())
|
||||
.await
|
||||
.expect("publish guarded fixture metadata");
|
||||
let generation = crate::disk::CurrentDeleteGeneration::from_file_info(&info).expect("capture object generation");
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let expected_error = if stale {
|
||||
let mut format_info = disk.format_info.write().await;
|
||||
format_info.id = Some(Uuid::new_v4());
|
||||
format_info.file_info = Some(tokio::fs::metadata(dir.path()).await.expect("local disk directory metadata"));
|
||||
format_info.last_check = Some(::time::OffsetDateTime::now_utc());
|
||||
drop(format_info);
|
||||
wrapper.set_disk_id_state(Some(Uuid::new_v4())).await;
|
||||
DiskError::DiskNotFound
|
||||
} else {
|
||||
wrapper.force_offline_for_test();
|
||||
DiskError::FaultyDisk
|
||||
};
|
||||
let object_dir = dir.path().join(volume).join(object);
|
||||
let metadata_path = object_dir.join(crate::disk::STORAGE_FORMAT_FILE);
|
||||
let before = tokio::fs::read(&metadata_path).await.expect("read original metadata");
|
||||
wrapper
|
||||
.supports_guarded_delete_versions()
|
||||
.await
|
||||
.expect("local binary capability is known even when the drive is unavailable");
|
||||
assert_eq!(
|
||||
wrapper.metrics_snapshot().total_errors_availability,
|
||||
0,
|
||||
"capability does not probe drive availability"
|
||||
);
|
||||
let transaction = Uuid::new_v4();
|
||||
let result = wrapper
|
||||
.delete_versions_guarded_recoverable(
|
||||
volume,
|
||||
vec![GuardedFileInfoVersions {
|
||||
group: FileInfoVersions {
|
||||
name: object.to_owned(),
|
||||
versions: vec![FileInfo {
|
||||
name: object.to_owned(),
|
||||
version_id: info.version_id,
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
},
|
||||
expected_current: generation,
|
||||
}],
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result, vec![Some(expected_error)], "mutation availability checks remain enforced");
|
||||
assert_eq!(
|
||||
tokio::fs::read(&metadata_path)
|
||||
.await
|
||||
.expect("metadata survives rejected mutation"),
|
||||
before
|
||||
);
|
||||
assert!(
|
||||
!object_dir.join(transaction.to_string()).exists(),
|
||||
"rejected mutation creates no recovery record"
|
||||
);
|
||||
assert_eq!(wrapper.metrics_snapshot().total_errors_availability, 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_disk_health_wrapper_counts_faulty_precheck_rejections() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -167,6 +167,12 @@ pub enum DiskError {
|
||||
#[error("method not allowed")]
|
||||
MethodNotAllowed,
|
||||
|
||||
#[error("conditional delete object generation changed")]
|
||||
ConditionalDeleteConflict,
|
||||
|
||||
#[error("guarded batch deletion is not supported by this disk")]
|
||||
GuardedDeleteUnsupported,
|
||||
|
||||
#[error("erasure write quorum")]
|
||||
ErasureWriteQuorum,
|
||||
|
||||
@@ -762,6 +768,8 @@ impl Clone for DiskError {
|
||||
DiskError::PartMissingOrCorrupt => DiskError::PartMissingOrCorrupt,
|
||||
DiskError::NoHealRequired => DiskError::NoHealRequired,
|
||||
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
|
||||
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
||||
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
||||
DiskError::ShortWrite => DiskError::ShortWrite,
|
||||
@@ -819,6 +827,8 @@ impl DiskError {
|
||||
DiskError::Timeout => 0x29,
|
||||
DiskError::InvalidPath => 0x2A,
|
||||
DiskError::RemoteClientUnavailable(_) => 0x2B,
|
||||
DiskError::ConditionalDeleteConflict => 0x2C,
|
||||
DiskError::GuardedDeleteUnsupported => 0x2D,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -867,6 +877,8 @@ impl DiskError {
|
||||
0x29 => Some(DiskError::Timeout),
|
||||
0x2A => Some(DiskError::InvalidPath),
|
||||
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
|
||||
0x2C => Some(DiskError::ConditionalDeleteConflict),
|
||||
0x2D => Some(DiskError::GuardedDeleteUnsupported),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -1645,6 +1657,21 @@ mod tests {
|
||||
assert!(rendered.contains("transport error: connection refused"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guarded_delete_errors_preserve_wire_and_io_types() {
|
||||
for (error, code) in [
|
||||
(DiskError::ConditionalDeleteConflict, 0x2C),
|
||||
(DiskError::GuardedDeleteUnsupported, 0x2D),
|
||||
] {
|
||||
assert_eq!(error.to_u32(), code);
|
||||
assert_eq!(DiskError::from_u32(code), Some(error.clone()));
|
||||
let wire: rustfs_protos::proto_gen::node_service::Error = error.clone().into();
|
||||
assert_eq!(DiskError::from(wire), error);
|
||||
let io_error: std::io::Error = error.clone().into();
|
||||
assert_eq!(DiskError::from(io_error), error);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_client_unavailable_wire_roundtrip_keeps_variant_and_detail() {
|
||||
let original = DiskError::RemoteClientUnavailable("dial tcp: connection refused".to_string());
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bucket-scoped observations distinguish exact metadata reads from walks.
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub(crate) struct Counts {
|
||||
pub walks: usize,
|
||||
pub reads: Vec<String>,
|
||||
}
|
||||
|
||||
fn observers() -> &'static Mutex<HashMap<String, Arc<Mutex<Counts>>>> {
|
||||
static OBSERVERS: OnceLock<Mutex<HashMap<String, Arc<Mutex<Counts>>>>> = OnceLock::new();
|
||||
OBSERVERS.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
pub(crate) struct Observer {
|
||||
bucket: String,
|
||||
counts: Arc<Mutex<Counts>>,
|
||||
}
|
||||
|
||||
impl Observer {
|
||||
pub fn new(bucket: &str) -> Self {
|
||||
let counts = Arc::new(Mutex::new(Counts::default()));
|
||||
assert!(
|
||||
observers()
|
||||
.lock()
|
||||
.expect("I/O observers lock")
|
||||
.insert(bucket.to_owned(), counts.clone())
|
||||
.is_none(),
|
||||
"each test must use its own bucket"
|
||||
);
|
||||
Self {
|
||||
bucket: bucket.to_owned(),
|
||||
counts,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn snapshot(&self) -> Counts {
|
||||
self.counts.lock().expect("I/O counts lock").clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Observer {
|
||||
fn drop(&mut self) {
|
||||
observers().lock().expect("I/O observers lock").remove(&self.bucket);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn observe(bucket: &str, object: Option<&str>) {
|
||||
let observer = observers().lock().expect("I/O observers lock").get(bucket).cloned();
|
||||
if let Some(observer) = observer {
|
||||
let mut counts = observer.lock().expect("I/O counts lock");
|
||||
match object {
|
||||
Some(object) => counts.reads.push(object.to_owned()),
|
||||
None => counts.walks += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
+1181
-11
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -26,6 +26,42 @@ pub struct ReplacementExecutionLease {
|
||||
// file while it is held would allow a second owner to lock a different inode.
|
||||
const EXECUTION_LOCK_FILE: &str = ".rustfs-replacement.lock";
|
||||
|
||||
/// A compatibility retry may only admit a disk that has not received data.
|
||||
/// The caller retains the mount descriptor and the replacement executor lease.
|
||||
pub(crate) async fn is_blank_target(root: PathBuf) -> Result<bool> {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let mut directories = vec![(root, 0usize)];
|
||||
let mut entries_left = 64usize;
|
||||
while let Some((directory, depth)) = directories.pop() {
|
||||
for entry in std::fs::read_dir(directory)? {
|
||||
let entry = entry?;
|
||||
let Some(remaining) = entries_left.checked_sub(1) else {
|
||||
return Ok(false);
|
||||
};
|
||||
entries_left = remaining;
|
||||
let kind = entry.file_type()?;
|
||||
let name = entry.file_name();
|
||||
if depth == 0 && name == EXECUTION_LOCK_FILE && kind.is_file() {
|
||||
if entry.metadata()?.len() != 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
} else if depth == 1 && name == crate::disk::HEALING_MARKER_PATH && kind.is_file() {
|
||||
if entry.metadata()?.len() > 256 {
|
||||
return Ok(false);
|
||||
}
|
||||
} else if kind.is_dir() && depth < 4 && (depth > 0 || name == crate::disk::RUSTFS_META_BUCKET) {
|
||||
directories.push((entry.path(), depth + 1));
|
||||
} else {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
|
||||
pub(super) async fn acquire(root: PathBuf) -> Result<Arc<ReplacementExecutionLease>> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
@@ -56,6 +92,53 @@ pub(super) async fn acquire(root: PathBuf) -> Result<Arc<ReplacementExecutionLea
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn blank_target_probe_rejects_data_symlinks_and_excessive_inventory() {
|
||||
let temp = tempfile::TempDir::new().expect("blank target root");
|
||||
let root = temp.path();
|
||||
let _lease = acquire(root.to_path_buf()).await.expect("replacement execution lease");
|
||||
let metadata = root.join(crate::disk::RUSTFS_META_BUCKET);
|
||||
std::fs::create_dir_all(metadata.join("tmp")).expect("empty system directories");
|
||||
std::fs::write(metadata.join(crate::disk::HEALING_MARKER_PATH), b"owner").expect("healing marker");
|
||||
assert!(
|
||||
is_blank_target(root.to_path_buf())
|
||||
.await
|
||||
.expect("blank target with control files")
|
||||
);
|
||||
|
||||
for path in [
|
||||
root.join("user-bucket"),
|
||||
metadata.join("tmp").join("payload"),
|
||||
metadata.join("format.json"),
|
||||
] {
|
||||
std::fs::write(&path, b"data").expect("residual content");
|
||||
assert!(!is_blank_target(root.to_path_buf()).await.expect("inspect residual content"));
|
||||
std::fs::remove_file(path).expect("remove residual fixture");
|
||||
}
|
||||
std::fs::create_dir(root.join("user-bucket")).expect("even an empty user bucket is not a fresh target");
|
||||
assert!(!is_blank_target(root.to_path_buf()).await.expect("inspect user bucket"));
|
||||
std::fs::remove_dir(root.join("user-bucket")).expect("remove bucket fixture");
|
||||
std::os::unix::fs::symlink(root, metadata.join("cycle")).expect("symlink fixture");
|
||||
assert!(
|
||||
!is_blank_target(root.to_path_buf())
|
||||
.await
|
||||
.expect("reject symlink without following it")
|
||||
);
|
||||
std::fs::remove_file(metadata.join("cycle")).expect("remove symlink fixture");
|
||||
std::fs::create_dir_all(metadata.join("a/b/c/d")).expect("excessive depth");
|
||||
assert!(!is_blank_target(root.to_path_buf()).await.expect("bounded directory depth"));
|
||||
std::fs::remove_dir_all(metadata.join("a")).expect("remove deep fixture");
|
||||
std::fs::remove_file(metadata.join(crate::disk::HEALING_MARKER_PATH)).expect("remove marker fixture");
|
||||
std::fs::remove_dir(metadata.join("tmp")).expect("remove empty fixture directory");
|
||||
for index in 0..62 {
|
||||
std::fs::create_dir(metadata.join(index.to_string())).expect("bounded empty directories");
|
||||
}
|
||||
assert!(is_blank_target(root.to_path_buf()).await.expect("exact inventory limit"));
|
||||
std::fs::create_dir(metadata.join("extra")).expect("one over inventory limit");
|
||||
assert!(!is_blank_target(root.to_path_buf()).await.expect("bounded inventory"));
|
||||
assert!(is_blank_target(root.join("missing")).await.is_err(), "missing root is not a blank disk");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_execution_lease_excludes_independent_openers() {
|
||||
let temp = tempfile::TempDir::new().expect("lease root");
|
||||
|
||||
@@ -24,6 +24,8 @@ pub mod format;
|
||||
pub mod fs;
|
||||
pub mod health_state;
|
||||
mod io_admission;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle_test_io;
|
||||
pub mod local;
|
||||
pub mod os;
|
||||
|
||||
@@ -115,7 +117,7 @@ impl RenameDataObservation {
|
||||
}
|
||||
|
||||
const QUOTA_MUTATION_FENCE_PREFIX: &str = "tmp/quota-mutation-fences/";
|
||||
pub(crate) const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
|
||||
pub const QUOTA_MUTATION_FENCE_METADATA_SUFFIX: &str = "quota-mutation-fence-token";
|
||||
|
||||
pub(crate) fn quota_mutation_fence_path(bucket: &str, object: &str) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
@@ -131,7 +133,7 @@ pub(crate) fn quota_mutation_fence_path(bucket: &str, object: &str) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn is_quota_mutation_fence_path(path: &str) -> bool {
|
||||
pub fn is_quota_mutation_fence_path(path: &str) -> bool {
|
||||
path.strip_prefix(QUOTA_MUTATION_FENCE_PREFIX)
|
||||
.is_some_and(|digest| digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
}
|
||||
@@ -231,6 +233,14 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// A read-only probe; the physical delete repeats the same leaf checks.
|
||||
pub async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.get_disk().is_plain_orphan_data_dir(volume, path).await,
|
||||
Self::Remote(disk) => disk.is_plain_orphan_data_dir(volume, path).await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_versions_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -347,6 +357,25 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_versions_guarded_recoverable_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
versions: Vec<GuardedFileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Vec<Option<DiskError>> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.delete_versions_guarded_with_namespace_owner(volume, versions, opts, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
let _owner = namespace_owner;
|
||||
disk.delete_versions_guarded_recoverable(volume, versions, opts).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cached_disk_id(&self) -> Option<Uuid> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.get_current_disk_id().await,
|
||||
@@ -357,6 +386,21 @@ impl Disk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for Disk {
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.supports_guarded_delete_versions().await,
|
||||
Self::Remote(disk) => disk.supports_guarded_delete_versions().await,
|
||||
}
|
||||
}
|
||||
async fn delete_versions_guarded_recoverable(
|
||||
&self,
|
||||
volume: &str,
|
||||
versions: Vec<GuardedFileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
) -> Vec<Option<DiskError>> {
|
||||
self.delete_versions_guarded_recoverable_with_namespace_owner(volume, versions, opts, None)
|
||||
.await
|
||||
}
|
||||
async fn delete_version_recoverable(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -379,6 +423,12 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.delete_versions_recoverable(volume, versions, opts).await,
|
||||
}
|
||||
}
|
||||
async fn has_delete_transaction_evidence(&self, volume: &str, object: &str) -> Result<bool> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.has_delete_transaction_evidence(volume, object).await,
|
||||
Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -622,6 +672,7 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
#[track_caller]
|
||||
async fn read_version(
|
||||
&self,
|
||||
_org_volume: &str,
|
||||
@@ -630,10 +681,20 @@ impl DiskAPI for Disk {
|
||||
version_id: &str,
|
||||
opts: &ReadOptions,
|
||||
) -> Result<FileInfo> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.read_version(_org_volume, volume, path, version_id, opts).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.read_version(_org_volume, volume, path, version_id, opts).await,
|
||||
let operation = async {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.read_version(_org_volume, volume, path, version_id, opts).await,
|
||||
Disk::Remote(remote_disk) => remote_disk.read_version(_org_volume, volume, path, version_id, opts).await,
|
||||
}
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
if crate::cluster::rpc::runtime_sources::current_read_version_request_caller().is_none() {
|
||||
let caller = crate::cluster::rpc::runtime_sources::read_version_request_caller_from_location(
|
||||
std::panic::Location::caller().file(),
|
||||
);
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_request_caller(caller, operation).await;
|
||||
}
|
||||
operation.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
@@ -1240,6 +1301,14 @@ impl Disk {
|
||||
Self::Remote(_) => Err(DiskError::other("replacement execution requires a local target")),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn is_blank_replacement_target(&self) -> Result<bool> {
|
||||
let Some(root) = self.replacement_mount_lease_root() else {
|
||||
return Ok(false);
|
||||
};
|
||||
let blank = local::is_blank_replacement_target(root).await?;
|
||||
Ok(blank && self.has_replacement_mount_lease())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn new_disk(ep: &Endpoint, opt: &DiskOption) -> Result<DiskStore> {
|
||||
@@ -1325,6 +1394,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
}
|
||||
async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>;
|
||||
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()>;
|
||||
#[track_caller]
|
||||
async fn read_version(
|
||||
&self,
|
||||
org_volume: &str,
|
||||
@@ -1410,6 +1480,11 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>;
|
||||
/// Exact-key existence owns its traversal state; unsupported peers cannot
|
||||
/// establish the absence of recovery obligations.
|
||||
async fn has_delete_transaction_evidence(&self, _volume: &str, _object: &str) -> Result<bool> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
_volume: &str,
|
||||
@@ -1419,6 +1494,18 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
) -> Result<DeleteRecoveryPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
/// A read-only capability check; unsupported implementations must fail closed.
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
Err(DiskError::GuardedDeleteUnsupported)
|
||||
}
|
||||
async fn delete_versions_guarded_recoverable(
|
||||
&self,
|
||||
_volume: &str,
|
||||
versions: Vec<GuardedFileInfoVersions>,
|
||||
_opts: DeleteOptions,
|
||||
) -> Vec<Option<DiskError>> {
|
||||
vec![Some(DiskError::GuardedDeleteUnsupported); versions.len()]
|
||||
}
|
||||
async fn delete_version_recoverable(
|
||||
&self,
|
||||
_volume: &str,
|
||||
@@ -1493,13 +1580,15 @@ where
|
||||
{
|
||||
validate_batch_read_version_item_count(req.items.len())?;
|
||||
let parallelism = batch_read_version_server_parallelism();
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let item_count = req.items.len();
|
||||
|
||||
let mut responses = stream::iter(req.items.into_iter().enumerate())
|
||||
.map(|(index, item)| async move {
|
||||
match disk
|
||||
.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts)
|
||||
.await
|
||||
{
|
||||
let read = disk.read_version(&item.org_volume, &item.volume, &item.path, &item.version_id, &req.opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_batch_read_version_item_context(index, item_count, read);
|
||||
match read.await {
|
||||
Ok(file_info) => BatchReadVersionResp {
|
||||
index,
|
||||
path: item.path,
|
||||
@@ -1649,6 +1738,117 @@ pub struct FileInfoVersions {
|
||||
pub free_versions: Vec<FileInfo>,
|
||||
}
|
||||
|
||||
/// The durable remote location and restore state of the observed generation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CurrentDeleteTransition {
|
||||
pub status: String,
|
||||
pub tier: String,
|
||||
pub remote_object: String,
|
||||
pub remote_version: Option<String>,
|
||||
pub version_state: rustfs_filemeta::TransitionVersionState,
|
||||
pub destination_id: Option<[u8; 32]>,
|
||||
pub remote_only: bool,
|
||||
}
|
||||
|
||||
/// The current object generation observed before a guarded batch delete.
|
||||
/// This transient value is never persisted in object metadata.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum CurrentDeleteGeneration {
|
||||
Absent,
|
||||
Present {
|
||||
version_id: Option<Uuid>,
|
||||
data_dir: Option<Uuid>,
|
||||
mod_time: OffsetDateTime,
|
||||
delete_marker: bool,
|
||||
etag: String,
|
||||
#[serde(default)]
|
||||
transition: Option<Box<CurrentDeleteTransition>>,
|
||||
},
|
||||
}
|
||||
|
||||
impl CurrentDeleteGeneration {
|
||||
pub fn from_file_info(fi: &FileInfo) -> Result<Self> {
|
||||
let mod_time = fi.mod_time.ok_or(DiskError::FileCorrupt)?;
|
||||
let etag = fi.get_etag().unwrap_or_default();
|
||||
let data_dir = fi.data_dir.filter(|id| !id.is_nil());
|
||||
let remote_only = fi.is_remote();
|
||||
if !fi.deleted && (etag.is_empty() || (data_dir.is_none() && !remote_only)) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
if remote_only && (fi.transition_tier.is_empty() || fi.transitioned_objname.is_empty()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let destination_id =
|
||||
crate::services::tier::tier::tier_destination_id_from_metadata(&fi.metadata).map_err(|_| DiskError::FileCorrupt)?;
|
||||
let remote_version = match fi.transition_version.as_deref() {
|
||||
Some(value) => (!value.is_empty() && !Uuid::parse_str(value).is_ok_and(|id| id.is_nil())).then(|| value.to_owned()),
|
||||
None => fi.transition_version_id.filter(|id| !id.is_nil()).map(|id| id.to_string()),
|
||||
};
|
||||
let transition = (!fi.transition_status.is_empty()
|
||||
|| !fi.transition_tier.is_empty()
|
||||
|| !fi.transitioned_objname.is_empty()
|
||||
|| remote_version.is_some()
|
||||
|| fi.transition_version_state != rustfs_filemeta::TransitionVersionState::Unknown
|
||||
|| destination_id.is_some())
|
||||
.then(|| {
|
||||
Box::new(CurrentDeleteTransition {
|
||||
status: fi.transition_status.clone(),
|
||||
tier: fi.transition_tier.clone(),
|
||||
remote_object: fi.transitioned_objname.clone(),
|
||||
remote_version,
|
||||
version_state: fi.transition_version_state,
|
||||
destination_id,
|
||||
remote_only,
|
||||
})
|
||||
});
|
||||
Ok(Self::Present {
|
||||
version_id: fi.version_id.filter(|id| !id.is_nil()),
|
||||
data_dir,
|
||||
mod_time,
|
||||
delete_marker: fi.deleted,
|
||||
etag,
|
||||
transition,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GuardedFileInfoVersions {
|
||||
pub group: FileInfoVersions,
|
||||
pub expected_current: CurrentDeleteGeneration,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct GuardedDeleteVersionsPayload {
|
||||
pub versions: Vec<GuardedFileInfoVersions>,
|
||||
pub opts: DeleteOptions,
|
||||
}
|
||||
|
||||
impl GuardedDeleteVersionsPayload {
|
||||
pub fn validate(&self) -> Result<()> {
|
||||
validate_delete_recovery_options(&self.opts)?;
|
||||
if self.versions.is_empty()
|
||||
|| self.versions.len() > 1000
|
||||
|| self
|
||||
.versions
|
||||
.iter()
|
||||
.try_fold(0_usize, |count, version| count.checked_add(version.group.versions.len()))
|
||||
.is_none_or(|count| count > 1000)
|
||||
|| self
|
||||
.versions
|
||||
.iter()
|
||||
.any(|version| version.group.name.starts_with('/') || version.group.versions.is_empty())
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl FileInfoVersions {
|
||||
pub fn find_version_index(&self, v: &str) -> Option<usize> {
|
||||
if v.is_empty() {
|
||||
@@ -1803,6 +2003,10 @@ pub struct DeleteOptions {
|
||||
pub delete_recovery_bucket_incarnation: Option<Uuid>,
|
||||
#[serde(default)]
|
||||
pub recoverable_delete: bool,
|
||||
/// Reclaim only plain erasure-part leaves, preserving every other entry.
|
||||
/// Keep `recursive` false so an older peer cannot widen this operation.
|
||||
#[serde(default)]
|
||||
pub orphan_data_dir: bool,
|
||||
}
|
||||
|
||||
pub const DELETE_RECOVERY_PROTOCOL_VERSION: u32 = 1;
|
||||
@@ -1833,6 +2037,7 @@ pub(crate) fn validate_delete_recovery_options(opts: &DeleteOptions) -> Result<(
|
||||
|| opts.undo_delete
|
||||
|| opts.recursive
|
||||
|| opts.immediate
|
||||
|| opts.orphan_data_dir
|
||||
|| opts.expected_delete_marker.is_some()
|
||||
|| opts.delete_recovery_bucket_incarnation.is_some_and(|id| id.is_nil())
|
||||
{
|
||||
@@ -2012,6 +2217,9 @@ pub enum DeleteRecoveryAction {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoveryRpcOperation {
|
||||
InspectOrphanDataDir {
|
||||
path: String,
|
||||
},
|
||||
Discover {
|
||||
object: Option<String>,
|
||||
cursor: Option<DeleteRecoveryCursor>,
|
||||
@@ -2037,6 +2245,9 @@ pub enum DeleteRecoveryRpcOperation {
|
||||
versions: Vec<FileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
},
|
||||
Exists {
|
||||
object: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl DeleteRecoveryRpcOperation {
|
||||
@@ -2057,11 +2268,13 @@ impl DeleteRecoveryRpcOperation {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoveryRpcReply {
|
||||
OrphanDataDir { plain: bool },
|
||||
Discovered(DeleteRecoveryPage),
|
||||
Inspected(DeleteRecoveryInspection),
|
||||
Settled(DeleteRecoveryInspection),
|
||||
AppliedVersion,
|
||||
AppliedVersions(Vec<Option<u32>>),
|
||||
Evidence { present: bool },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -2177,6 +2390,243 @@ pub fn count_part_not_success(part_errs: &[usize]) -> usize {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn guarded_delete_generation_boxing_preserves_legacy_wire_bytes() {
|
||||
#[derive(Serialize)]
|
||||
struct LegacyPresent {
|
||||
version_id: Option<Uuid>,
|
||||
data_dir: Option<Uuid>,
|
||||
mod_time: OffsetDateTime,
|
||||
delete_marker: bool,
|
||||
etag: String,
|
||||
transition: Option<CurrentDeleteTransition>,
|
||||
}
|
||||
|
||||
assert!(std::mem::size_of::<CurrentDeleteGeneration>() < std::mem::size_of::<LegacyPresent>());
|
||||
let mut fi = FileInfo {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
data_dir: Some(Uuid::from_u128(1)),
|
||||
..Default::default()
|
||||
};
|
||||
fi.metadata.insert("etag".into(), "opaque-etag".into());
|
||||
let plain = CurrentDeleteGeneration::from_file_info(&fi).expect("local generation");
|
||||
fi.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_owned();
|
||||
fi.transition_tier = "WARM".into();
|
||||
fi.transitioned_objname = "remote/object".into();
|
||||
fi.transition_version = Some("remote-version".into());
|
||||
let remote = CurrentDeleteGeneration::from_file_info(&fi).expect("transition generation");
|
||||
for current in [CurrentDeleteGeneration::Absent, plain, remote] {
|
||||
let (legacy_messagepack, legacy_json) = match ¤t {
|
||||
CurrentDeleteGeneration::Absent => (
|
||||
rmp_serde::to_vec_named("Absent").expect("legacy absent MessagePack"),
|
||||
serde_json::to_vec("Absent").expect("legacy absent JSON"),
|
||||
),
|
||||
CurrentDeleteGeneration::Present {
|
||||
version_id,
|
||||
data_dir,
|
||||
mod_time,
|
||||
delete_marker,
|
||||
etag,
|
||||
transition,
|
||||
} => {
|
||||
let legacy = std::collections::BTreeMap::from([(
|
||||
"Present",
|
||||
LegacyPresent {
|
||||
version_id: *version_id,
|
||||
data_dir: *data_dir,
|
||||
mod_time: *mod_time,
|
||||
delete_marker: *delete_marker,
|
||||
etag: etag.clone(),
|
||||
transition: transition.as_deref().cloned(),
|
||||
},
|
||||
)]);
|
||||
(
|
||||
rmp_serde::to_vec_named(&legacy).expect("legacy inline MessagePack"),
|
||||
serde_json::to_vec(&legacy).expect("legacy inline JSON"),
|
||||
)
|
||||
}
|
||||
};
|
||||
assert_eq!(rmp_serde::to_vec_named(¤t).expect("current MessagePack"), legacy_messagepack);
|
||||
assert_eq!(serde_json::to_vec(¤t).expect("current JSON"), legacy_json);
|
||||
assert_eq!(
|
||||
rustfs_protos::decode_delete_recovery_payload::<CurrentDeleteGeneration>(&legacy_messagepack)
|
||||
.expect("existing payload remains readable"),
|
||||
current
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guarded_delete_generation_requires_reliable_current_identity() {
|
||||
let mut fi = FileInfo::default();
|
||||
assert_eq!(CurrentDeleteGeneration::from_file_info(&fi), Err(DiskError::FileCorrupt));
|
||||
fi.mod_time = Some(OffsetDateTime::UNIX_EPOCH);
|
||||
fi.data_dir = Some(Uuid::new_v4());
|
||||
assert_eq!(CurrentDeleteGeneration::from_file_info(&fi), Err(DiskError::FileCorrupt));
|
||||
fi.metadata.insert("etag".into(), "opaque-etag".into());
|
||||
fi.version_id = Some(Uuid::nil());
|
||||
assert!(matches!(
|
||||
CurrentDeleteGeneration::from_file_info(&fi),
|
||||
Ok(CurrentDeleteGeneration::Present { version_id: None, .. })
|
||||
));
|
||||
fi.data_dir = Some(Uuid::nil());
|
||||
assert_eq!(CurrentDeleteGeneration::from_file_info(&fi), Err(DiskError::FileCorrupt));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guarded_delete_generation_accepts_remote_only_and_legacy_unversioned_identity() {
|
||||
let mut fi = FileInfo {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_owned(),
|
||||
transition_tier: "WARM".into(),
|
||||
transitioned_objname: "remote/object".into(),
|
||||
..Default::default()
|
||||
};
|
||||
fi.metadata.insert("etag".into(), "opaque-etag".into());
|
||||
for version in [None, Some(String::new()), Some(Uuid::nil().to_string())] {
|
||||
fi.transition_version = version;
|
||||
for data_dir in [None, Some(Uuid::nil())] {
|
||||
fi.data_dir = data_dir;
|
||||
assert!(
|
||||
matches!(
|
||||
CurrentDeleteGeneration::from_file_info(&fi),
|
||||
Ok(CurrentDeleteGeneration::Present {
|
||||
data_dir: None,
|
||||
transition: Some(transition),
|
||||
..
|
||||
}) if transition.remote_version.is_none()
|
||||
&& transition.version_state == rustfs_filemeta::TransitionVersionState::Unknown
|
||||
&& transition.destination_id.is_none()
|
||||
&& transition.remote_only
|
||||
),
|
||||
"legacy unversioned remote metadata remains usable"
|
||||
);
|
||||
}
|
||||
}
|
||||
fi.transition_version = None;
|
||||
fi.transition_version_id = Some(Uuid::nil());
|
||||
CurrentDeleteGeneration::from_file_info(&fi).expect("nil legacy remote UUID means no remote version");
|
||||
fi.transition_status.clear();
|
||||
assert_eq!(
|
||||
CurrentDeleteGeneration::from_file_info(&fi),
|
||||
Err(DiskError::FileCorrupt),
|
||||
"local payload metadata still requires a data directory"
|
||||
);
|
||||
fi.transition_status = rustfs_filemeta::TRANSITION_COMPLETE.to_owned();
|
||||
fi.transition_tier.clear();
|
||||
assert_eq!(
|
||||
CurrentDeleteGeneration::from_file_info(&fi),
|
||||
Err(DiskError::FileCorrupt),
|
||||
"remote-only identity requires its tier"
|
||||
);
|
||||
fi.transition_tier = "WARM".into();
|
||||
fi.transitioned_objname.clear();
|
||||
assert_eq!(
|
||||
CurrentDeleteGeneration::from_file_info(&fi),
|
||||
Err(DiskError::FileCorrupt),
|
||||
"remote-only identity requires its remote object"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guarded_delete_remote_transition_tuple_survives_rpc_codec() {
|
||||
use rustfs_utils::http::metadata_compat::{SUFFIX_TRANSITION_TIER_DESTINATION_ID, insert_str};
|
||||
let mut fi = FileInfo {
|
||||
name: "object".into(),
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
transition_status: rustfs_filemeta::TRANSITION_COMPLETE.to_owned(),
|
||||
transition_tier: "WARM".into(),
|
||||
transitioned_objname: "opaque/object+name".into(),
|
||||
transition_version: Some("opaque/version+id==".into()),
|
||||
transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
..Default::default()
|
||||
};
|
||||
fi.metadata.insert("etag".into(), "opaque-etag".into());
|
||||
insert_str(&mut fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID, "ab".repeat(32));
|
||||
let generation = CurrentDeleteGeneration::from_file_info(&fi).expect("capture complete remote identity");
|
||||
let payload = GuardedDeleteVersionsPayload {
|
||||
versions: vec![GuardedFileInfoVersions {
|
||||
group: FileInfoVersions {
|
||||
name: "object".into(),
|
||||
versions: vec![fi.clone()],
|
||||
..Default::default()
|
||||
},
|
||||
expected_current: generation.clone(),
|
||||
}],
|
||||
opts: DeleteOptions {
|
||||
old_data_dir: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
let encoded = rmp_serde::to_vec_named(&payload).expect("encode remote guarded request");
|
||||
let decoded: GuardedDeleteVersionsPayload =
|
||||
rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode remote guarded request");
|
||||
assert_eq!(decoded.versions[0].expected_current, generation);
|
||||
assert!(
|
||||
matches!(&generation, CurrentDeleteGeneration::Present { transition: Some(transition), .. }
|
||||
if transition.status == rustfs_filemeta::TRANSITION_COMPLETE
|
||||
&& transition.tier == "WARM"
|
||||
&& transition.remote_object == "opaque/object+name"
|
||||
&& transition.remote_version.as_deref() == Some("opaque/version+id==")
|
||||
&& transition.version_state == rustfs_filemeta::TransitionVersionState::Exact
|
||||
&& transition.destination_id == Some([0xab; 32])
|
||||
&& transition.remote_only)
|
||||
);
|
||||
insert_str(&mut fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID, format!("a{}", "€".repeat(21)));
|
||||
assert_eq!(
|
||||
CurrentDeleteGeneration::from_file_info(&fi),
|
||||
Err(DiskError::FileCorrupt),
|
||||
"64-byte non-ASCII backend identities are rejected without panicking"
|
||||
);
|
||||
insert_str(&mut fi.metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID, "ab".repeat(32));
|
||||
fi.metadata
|
||||
.insert(format!("x-minio-internal-{SUFFIX_TRANSITION_TIER_DESTINATION_ID}"), "cd".repeat(32));
|
||||
assert_eq!(
|
||||
CurrentDeleteGeneration::from_file_info(&fi),
|
||||
Err(DiskError::FileCorrupt),
|
||||
"conflicting backend identities fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn guarded_delete_payload_rejects_malformed_or_unbounded_batches() {
|
||||
let group = GuardedFileInfoVersions {
|
||||
group: FileInfoVersions {
|
||||
name: "object".into(),
|
||||
versions: vec![FileInfo::default()],
|
||||
..Default::default()
|
||||
},
|
||||
expected_current: CurrentDeleteGeneration::Absent,
|
||||
};
|
||||
let mut payload = GuardedDeleteVersionsPayload {
|
||||
versions: vec![group],
|
||||
opts: DeleteOptions {
|
||||
old_data_dir: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
},
|
||||
};
|
||||
payload.validate().expect("valid guarded batch");
|
||||
let encoded = rmp_serde::to_vec_named(&payload).expect("serialize guarded batch");
|
||||
let decoded: GuardedDeleteVersionsPayload =
|
||||
rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode guarded batch");
|
||||
decoded.validate().expect("round-trip guarded batch remains valid");
|
||||
let mut trailing = encoded;
|
||||
trailing.push(0xc0);
|
||||
assert!(rustfs_protos::decode_delete_recovery_payload::<GuardedDeleteVersionsPayload>(&trailing).is_err());
|
||||
let mut unknown = serde_json::to_value(&payload).expect("serialize guarded batch to map");
|
||||
unknown
|
||||
.as_object_mut()
|
||||
.expect("payload is a map")
|
||||
.insert("unrecognized".into(), true.into());
|
||||
assert!(serde_json::from_value::<GuardedDeleteVersionsPayload>(unknown).is_err());
|
||||
payload.versions[0].group.versions.resize_with(1001, FileInfo::default);
|
||||
assert_eq!(payload.validate(), Err(DiskError::InvalidPath));
|
||||
payload.versions[0].group.versions.truncate(1);
|
||||
payload.opts.undo_delete = true;
|
||||
assert_eq!(payload.validate(), Err(DiskError::FileCorrupt));
|
||||
}
|
||||
|
||||
use endpoint::Endpoint;
|
||||
use local::LocalDisk;
|
||||
use tokio::fs;
|
||||
@@ -2354,6 +2804,7 @@ mod tests {
|
||||
expected_delete_marker: None,
|
||||
delete_recovery_bucket_incarnation: None,
|
||||
recoverable_delete: false,
|
||||
orphan_data_dir: false,
|
||||
};
|
||||
|
||||
assert!(opts.recursive);
|
||||
@@ -2807,6 +3258,88 @@ mod delete_recovery_timeout_tests {
|
||||
mod delete_recovery_wire_limit_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn boxed_current_delete_transition_preserves_existing_wire_payload() {
|
||||
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
enum PriorGeneration {
|
||||
Present {
|
||||
version_id: Option<Uuid>,
|
||||
data_dir: Option<Uuid>,
|
||||
mod_time: OffsetDateTime,
|
||||
delete_marker: bool,
|
||||
etag: String,
|
||||
#[serde(default)]
|
||||
transition: Option<CurrentDeleteTransition>,
|
||||
},
|
||||
}
|
||||
|
||||
let version_id = Some(Uuid::from_u128(1));
|
||||
let data_dir = Some(Uuid::from_u128(2));
|
||||
let mod_time = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1);
|
||||
for transition in [
|
||||
None,
|
||||
Some(CurrentDeleteTransition {
|
||||
status: rustfs_filemeta::TRANSITION_COMPLETE.to_owned(),
|
||||
tier: "WARM".into(),
|
||||
remote_object: "opaque/object+name".into(),
|
||||
remote_version: Some("opaque/version+id==".into()),
|
||||
version_state: rustfs_filemeta::TransitionVersionState::Exact,
|
||||
destination_id: Some([0xab; 32]),
|
||||
remote_only: true,
|
||||
}),
|
||||
] {
|
||||
let prior = PriorGeneration::Present {
|
||||
version_id,
|
||||
data_dir,
|
||||
mod_time,
|
||||
delete_marker: false,
|
||||
etag: "opaque-etag".into(),
|
||||
transition: transition.clone(),
|
||||
};
|
||||
let generation = CurrentDeleteGeneration::Present {
|
||||
version_id,
|
||||
data_dir,
|
||||
mod_time,
|
||||
delete_marker: false,
|
||||
etag: "opaque-etag".into(),
|
||||
transition: transition.map(Box::new),
|
||||
};
|
||||
let prior_encoded = rmp_serde::to_vec_named(&prior).expect("encode the previous unboxed generation shape");
|
||||
let encoded = rmp_serde::to_vec_named(&generation).expect("encode boxed transition generation");
|
||||
assert_eq!(encoded, prior_encoded);
|
||||
let decoded: CurrentDeleteGeneration =
|
||||
rustfs_protos::decode_delete_recovery_payload(&prior_encoded).expect("decode the previous generation payload");
|
||||
assert_eq!(decoded, generation);
|
||||
let prior_decoded: PriorGeneration =
|
||||
rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode boxed transition with the previous shape");
|
||||
assert_eq!(prior_decoded, prior);
|
||||
if matches!(generation, CurrentDeleteGeneration::Present { transition: None, .. }) {
|
||||
#[derive(Serialize)]
|
||||
enum GenerationWithoutTransition {
|
||||
Present {
|
||||
version_id: Option<Uuid>,
|
||||
data_dir: Option<Uuid>,
|
||||
mod_time: OffsetDateTime,
|
||||
delete_marker: bool,
|
||||
etag: String,
|
||||
},
|
||||
}
|
||||
let legacy = GenerationWithoutTransition::Present {
|
||||
version_id,
|
||||
data_dir,
|
||||
mod_time,
|
||||
delete_marker: false,
|
||||
etag: "opaque-etag".into(),
|
||||
};
|
||||
let legacy_encoded = rmp_serde::to_vec_named(&legacy).expect("encode a generation without the transition field");
|
||||
let legacy_decoded: CurrentDeleteGeneration =
|
||||
rustfs_protos::decode_delete_recovery_payload(&legacy_encoded).expect("default an omitted transition field");
|
||||
assert_eq!(legacy_decoded, generation);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn boxed_apply_version_preserves_existing_wire_payload() {
|
||||
#[derive(Serialize)]
|
||||
@@ -2964,3 +3497,87 @@ mod delete_recovery_adoption_tests {
|
||||
assert!(!journal.matches_adoption_source(&journal.original_versions));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod orphan_data_dir_protocol_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn delete_recovery_exists_roundtrips_without_cursor_ownership() {
|
||||
let operation = DeleteRecoveryRpcOperation::Exists {
|
||||
object: "object".to_owned(),
|
||||
};
|
||||
assert_eq!(
|
||||
operation.max_wire_payload_bytes(),
|
||||
rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES
|
||||
);
|
||||
let encoded = rmp_serde::to_vec_named(&operation).expect("encode exact-key existence probe");
|
||||
assert!(matches!(
|
||||
rustfs_protos::decode_delete_recovery_payload::<DeleteRecoveryRpcOperation>(&encoded).expect("decode existence probe"),
|
||||
DeleteRecoveryRpcOperation::Exists { object } if object == "object"
|
||||
));
|
||||
for present in [false, true] {
|
||||
let encoded = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::Evidence { present }).expect("encode evidence reply");
|
||||
assert!(matches!(
|
||||
rustfs_protos::decode_delete_recovery_payload::<DeleteRecoveryRpcReply>(&encoded).expect("decode evidence reply"),
|
||||
DeleteRecoveryRpcReply::Evidence { present: decoded } if decoded == present
|
||||
));
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
enum OlderOperation {
|
||||
Discover {
|
||||
_object: Option<String>,
|
||||
_cursor: Option<DeleteRecoveryCursor>,
|
||||
_limit: u32,
|
||||
},
|
||||
}
|
||||
assert!(
|
||||
rustfs_protos::decode_delete_recovery_payload::<OlderOperation>(&encoded).is_err(),
|
||||
"an older peer must reject existence probing instead of reporting absence"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_probe_roundtrips_without_mutation_options() {
|
||||
let path = format!("object/{}", Uuid::new_v4());
|
||||
let operation = DeleteRecoveryRpcOperation::InspectOrphanDataDir { path: path.clone() };
|
||||
assert_eq!(
|
||||
operation.max_wire_payload_bytes(),
|
||||
rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES
|
||||
);
|
||||
let encoded = rmp_serde::to_vec_named(&operation).expect("encode orphan probe");
|
||||
let decoded: DeleteRecoveryRpcOperation = rustfs_protos::decode_delete_recovery_payload(&encoded).expect("decode probe");
|
||||
assert!(matches!(decoded, DeleteRecoveryRpcOperation::InspectOrphanDataDir { path: decoded } if decoded == path));
|
||||
let reply = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::OrphanDataDir { plain: true }).expect("encode probe reply");
|
||||
assert!(matches!(
|
||||
rustfs_protos::decode_delete_recovery_payload::<DeleteRecoveryRpcReply>(&reply).expect("decode probe reply"),
|
||||
DeleteRecoveryRpcReply::OrphanDataDir { plain: true }
|
||||
));
|
||||
#[derive(Deserialize)]
|
||||
enum OlderOperation {
|
||||
Inspect { _object: String, _transaction: Uuid },
|
||||
}
|
||||
assert!(
|
||||
rustfs_protos::decode_delete_recovery_payload::<OlderOperation>(&encoded).is_err(),
|
||||
"older peers must reject the additive probe"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_delete_is_nonrecursive_when_older_peer_ignores_new_option() {
|
||||
#[derive(Deserialize)]
|
||||
struct OlderDeleteOptions {
|
||||
recursive: bool,
|
||||
immediate: bool,
|
||||
}
|
||||
let opts = DeleteOptions {
|
||||
orphan_data_dir: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
};
|
||||
let encoded = serde_json::to_vec(&opts).expect("encode leaf deletion");
|
||||
let old: OlderDeleteOptions = serde_json::from_slice(&encoded).expect("older options ignore additive field");
|
||||
assert!(!old.recursive, "ignoring the new option must never broaden deletion");
|
||||
assert!(old.immediate);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,19 @@ use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_errs};
|
||||
use crate::error::StorageError;
|
||||
use rustfs_protos::proto_gen::node_service::Error as WireError;
|
||||
|
||||
#[test]
|
||||
fn conditional_delete_errors_preserve_disk_storage_and_wire_identity() {
|
||||
for disk_error in [DiskError::ConditionalDeleteConflict, DiskError::GuardedDeleteUnsupported] {
|
||||
let storage: StorageError = disk_error.clone().into();
|
||||
let raw_code = storage.to_u32();
|
||||
assert_eq!(StorageError::from_u32(raw_code), Some(storage.clone()));
|
||||
assert_eq!(storage.narrow_to_disk().expect("guard errors must narrow"), disk_error);
|
||||
let wire: WireError = disk_error.clone().into();
|
||||
let decoded: DiskError = wire.into();
|
||||
assert_eq!(decoded, disk_error, "guard errors must survive peer RPC");
|
||||
}
|
||||
}
|
||||
|
||||
/// The variants heal / replication / quorum repair match on, and that must
|
||||
/// survive the DiskError → StorageError → DiskError loop unchanged.
|
||||
fn heal_matched_disk_variants() -> Vec<DiskError> {
|
||||
|
||||
@@ -290,6 +290,10 @@ pub enum StorageError {
|
||||
ConfigNotFound,
|
||||
#[error("Precondition failed")]
|
||||
PreconditionFailed,
|
||||
#[error("Object generation changed during conditional deletion")]
|
||||
ConditionalDeleteConflict,
|
||||
#[error("The storage peer does not support guarded deletion")]
|
||||
GuardedDeleteUnsupported,
|
||||
#[error("Not modified")]
|
||||
NotModified,
|
||||
#[error("Invalid range specified: {0}")]
|
||||
@@ -469,6 +473,8 @@ impl From<DiskError> for StorageError {
|
||||
DiskError::SourceStalled => StorageError::SourceStalled,
|
||||
DiskError::Timeout => StorageError::Timeout,
|
||||
DiskError::InvalidPath => StorageError::InvalidPath,
|
||||
DiskError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
|
||||
DiskError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail),
|
||||
}
|
||||
}
|
||||
@@ -512,6 +518,8 @@ impl StorageError {
|
||||
StorageError::SourceStalled => DiskError::SourceStalled,
|
||||
StorageError::Timeout => DiskError::Timeout,
|
||||
StorageError::InvalidPath => DiskError::InvalidPath,
|
||||
StorageError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
|
||||
StorageError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
|
||||
StorageError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
||||
StorageError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
||||
StorageError::TooManyOpenFiles => DiskError::TooManyOpenFiles,
|
||||
@@ -758,6 +766,8 @@ impl Clone for StorageError {
|
||||
StorageError::InsufficientReadQuorum(a, b) => StorageError::InsufficientReadQuorum(a.clone(), b.clone()),
|
||||
StorageError::InsufficientWriteQuorum(a, b) => StorageError::InsufficientWriteQuorum(a.clone(), b.clone()),
|
||||
StorageError::PreconditionFailed => StorageError::PreconditionFailed,
|
||||
StorageError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
|
||||
StorageError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
|
||||
StorageError::NotModified => StorageError::NotModified,
|
||||
StorageError::InvalidPartNumber(a) => StorageError::InvalidPartNumber(*a),
|
||||
StorageError::InvalidRangeSpec(a) => StorageError::InvalidRangeSpec(a.clone()),
|
||||
@@ -871,6 +881,8 @@ impl StorageError {
|
||||
StorageError::InsufficientReadQuorum(_, _) => StorageErrorCode::InsufficientReadQuorum,
|
||||
StorageError::InsufficientWriteQuorum(_, _) => StorageErrorCode::InsufficientWriteQuorum,
|
||||
StorageError::PreconditionFailed => StorageErrorCode::PreconditionFailed,
|
||||
StorageError::ConditionalDeleteConflict => StorageErrorCode::ConditionalDeleteConflict,
|
||||
StorageError::GuardedDeleteUnsupported => StorageErrorCode::GuardedDeleteUnsupported,
|
||||
StorageError::EntityTooSmall(_, _, _) => StorageErrorCode::EntityTooSmall,
|
||||
StorageError::EntityTooLarge(_, _) => StorageErrorCode::EntityTooLarge,
|
||||
StorageError::InvalidRangeSpec(_) => StorageErrorCode::InvalidRangeSpec,
|
||||
@@ -992,6 +1004,8 @@ impl StorageError {
|
||||
Some(StorageError::InsufficientWriteQuorum(Default::default(), Default::default()))
|
||||
}
|
||||
StorageErrorCode::PreconditionFailed => Some(StorageError::PreconditionFailed),
|
||||
StorageErrorCode::ConditionalDeleteConflict => Some(StorageError::ConditionalDeleteConflict),
|
||||
StorageErrorCode::GuardedDeleteUnsupported => Some(StorageError::GuardedDeleteUnsupported),
|
||||
StorageErrorCode::EntityTooSmall => {
|
||||
Some(StorageError::EntityTooSmall(Default::default(), Default::default(), Default::default()))
|
||||
}
|
||||
|
||||
@@ -898,6 +898,10 @@ pub enum WriteCompletion {
|
||||
/// Drain the rename fan-out before returning. Minority failures still heal
|
||||
/// after a successful quorum commit; this does not require every disk to succeed.
|
||||
TailDrained,
|
||||
/// Drain the rename fan-out and return system-metadata repair to the
|
||||
/// caller's existing deployment/transaction owner. This does not grant
|
||||
/// write or heal authority and does not change the quorum requirement.
|
||||
TailDrainedWithOwnerRepair,
|
||||
}
|
||||
|
||||
/// Storage-owned write mode inherited by physical rewrites. This selects the
|
||||
@@ -942,6 +946,11 @@ pub struct ObjectOptions {
|
||||
/// serialized or copied into object metadata.
|
||||
#[doc(hidden)]
|
||||
pub delete_mutation_owner: Option<Arc<dyn Send + Sync>>,
|
||||
/// Pool-local generations for admitted current-object conditional actions.
|
||||
/// Ordinary actions execute in a separate phase without this transient map.
|
||||
/// This must never be populated from client metadata.
|
||||
#[doc(hidden)]
|
||||
pub conditional_delete_generations: Option<Arc<HashMap<String, crate::disk::CurrentDeleteGeneration>>>,
|
||||
/// Internal read-only inspection must not enqueue metadata or payload repairs.
|
||||
pub suppress_read_repair: bool,
|
||||
/// Control-plane writers that immediately read or CAS the same namespace
|
||||
@@ -1006,6 +1015,9 @@ pub struct ObjectOptions {
|
||||
/// Proves an upper layer holds the bucket lifecycle sentinel. A separate
|
||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||
/// Retains the actual lifecycle guard for an owned multipart commit. A
|
||||
/// lock-loss fence alone does not keep a cancelled caller's lock alive.
|
||||
pub bucket_lifecycle_lock_owner: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
pub replication_request: bool,
|
||||
/// True when the inbound request carried the
|
||||
/// `{x-rustfs-,x-minio-}source-proxy-request` header family with the
|
||||
@@ -1163,8 +1175,10 @@ impl std::fmt::Debug for ObjectOptions {
|
||||
.field("delete_replication", &self.delete_replication.is_some())
|
||||
.field("delete_replication_config_snapshot", &self.delete_replication_config_snapshot)
|
||||
.field("delete_mutation_owner", &self.delete_mutation_owner.is_some())
|
||||
.field("conditional_delete_generations", &self.conditional_delete_generations.is_some())
|
||||
.field("namespace_lock_fence", &self.namespace_lock_fence.is_some())
|
||||
.field("bucket_lifecycle_lock_fence", &self.bucket_lifecycle_lock_fence.is_some())
|
||||
.field("bucket_lifecycle_lock_owner", &self.bucket_lifecycle_lock_owner.is_some())
|
||||
.field("replication_request", &self.replication_request)
|
||||
.field("proxy_request", &self.proxy_request)
|
||||
.field("proxy_header_set", &self.proxy_header_set)
|
||||
|
||||
@@ -49,6 +49,7 @@ use crate::layout::endpoints::{EndpointServerPools, SetupType};
|
||||
use crate::object_api::ObjectEncryptionResolver;
|
||||
use crate::services::event_notification::EventNotifier;
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::store::delete_recovery_runtime::DeleteRecoveryHints;
|
||||
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
|
||||
use s3s::region::Region;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
@@ -240,6 +241,7 @@ pub struct InstanceContext {
|
||||
namespace_commit_generation: AtomicU64,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
delete_recovery_hints: DeleteRecoveryHints,
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
#[cfg(test)]
|
||||
suppress_tier_delete_journal_recovery: bool,
|
||||
@@ -294,6 +296,7 @@ impl InstanceContext {
|
||||
namespace_commits: AtomicU64::new(0),
|
||||
namespace_commit_generation: AtomicU64::new(0),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
delete_recovery_hints: DeleteRecoveryHints::default(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
#[cfg(test)]
|
||||
suppress_tier_delete_journal_recovery: false,
|
||||
@@ -728,6 +731,10 @@ impl InstanceContext {
|
||||
self.background_cancel_token.get_or_init(|| token).clone()
|
||||
}
|
||||
|
||||
pub(crate) fn delete_recovery_hints(&self) -> &DeleteRecoveryHints {
|
||||
&self.delete_recovery_hints
|
||||
}
|
||||
|
||||
pub(crate) fn mark_tier_delete_journal_recovery_started(&self, store_id: Uuid) -> bool {
|
||||
#[cfg(test)]
|
||||
if self.suppress_tier_delete_journal_recovery {
|
||||
|
||||
@@ -546,6 +546,17 @@ pub fn acquire_cross_pool_fence_fleet_proof() -> Option<CrossPoolFenceFleetProof
|
||||
acquire_fleet_capability_proof_from(&state, expected_topology, Instant::now()).map(CrossPoolFenceFleetProofToken)
|
||||
}
|
||||
|
||||
/// Startup layout identity remains available when peers or the live fleet proof are unavailable.
|
||||
pub fn configured_fleet_topology_fingerprint() -> Option<&'static str> {
|
||||
let state = cross_pool_fence_fleet_proof_slot()
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.topology_conflict {
|
||||
return None;
|
||||
}
|
||||
REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get().map(String::as_str)
|
||||
}
|
||||
|
||||
pub fn cross_pool_fence_fleet_proof_matches(proof: &CrossPoolFenceFleetProofToken) -> bool {
|
||||
fleet_capability_proof_matches(cross_pool_fence_fleet_proof_slot(), &proof.0)
|
||||
}
|
||||
@@ -970,8 +981,8 @@ pub(crate) fn install_cross_pool_fence_fleet_proof_for_test() {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct CrossPoolFenceFleetProofGuard {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub struct CrossPoolFenceFleetProofGuard {
|
||||
previous_proof: Option<FleetCapabilityProof>,
|
||||
previous_topology_conflict: bool,
|
||||
previous_journal_proof: Option<FleetCapabilityProof>,
|
||||
@@ -980,7 +991,7 @@ pub(crate) struct CrossPoolFenceFleetProofGuard {
|
||||
previous_decommission_topology_conflict: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
impl Drop for CrossPoolFenceFleetProofGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut state = cross_pool_fence_fleet_proof_slot()
|
||||
@@ -1020,8 +1031,19 @@ impl Drop for CrossPoolFenceFleetProofGuard {
|
||||
|
||||
/// Temporarily revoke the test proof so activation paths can exercise their
|
||||
/// fail-closed behavior without changing the process-wide topology binding.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceFleetProofGuard {
|
||||
revoke_cross_pool_fence_fleet_proof_for_test(true)
|
||||
}
|
||||
|
||||
/// Simulate peer unavailability while retaining the configured topology identity.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceFleetProofGuard {
|
||||
pub(crate) fn without_live_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceFleetProofGuard {
|
||||
revoke_cross_pool_fence_fleet_proof_for_test(false)
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
fn revoke_cross_pool_fence_fleet_proof_for_test(topology_conflict: bool) -> CrossPoolFenceFleetProofGuard {
|
||||
let mut state = cross_pool_fence_fleet_proof_slot()
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
@@ -1045,21 +1067,21 @@ pub(crate) fn without_cross_pool_fence_fleet_proof_for_test() -> CrossPoolFenceF
|
||||
state.draining_generation = Some(proof.generation);
|
||||
}
|
||||
}
|
||||
state.topology_conflict = true;
|
||||
state.topology_conflict |= topology_conflict;
|
||||
if let Some(proof) = journal_state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
if !proof.generation.is_drained() {
|
||||
journal_state.draining_generation = Some(proof.generation);
|
||||
}
|
||||
}
|
||||
journal_state.topology_conflict = true;
|
||||
journal_state.topology_conflict |= topology_conflict;
|
||||
if let Some(proof) = decommission_state.proof.take() {
|
||||
proof.generation.revoke();
|
||||
if !proof.generation.is_drained() {
|
||||
decommission_state.draining_generation = Some(proof.generation);
|
||||
}
|
||||
}
|
||||
decommission_state.topology_conflict = true;
|
||||
decommission_state.topology_conflict |= topology_conflict;
|
||||
guard
|
||||
}
|
||||
|
||||
|
||||
@@ -18,8 +18,8 @@ use super::{
|
||||
encode_rebalance_stop_propagation_record,
|
||||
};
|
||||
use crate::core::pools::{
|
||||
PoolMeta, PoolRebalanceActivationFence, acquire_pool_activation_fleet_proof, acquire_pool_rebalance_activation_locks,
|
||||
pool_meta_has_active_decommission,
|
||||
PoolMeta, PoolRebalanceActivationFence, PoolRebalanceWorkerFence, acquire_pool_activation_fleet_proof,
|
||||
acquire_pool_rebalance_activation_locks, acquire_pool_rebalance_worker_locks, pool_meta_has_active_decommission,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -77,10 +77,33 @@ fn ensure_rebalance_activation_pool_meta_allowed(meta: &PoolMeta) -> Result<()>
|
||||
}
|
||||
|
||||
pub(super) enum RebalanceWorkerActivationFence {
|
||||
Ready(Box<PoolRebalanceActivationFence>),
|
||||
Ready(Box<PoolRebalanceWorkerFence>),
|
||||
NotStartedTerminal,
|
||||
}
|
||||
|
||||
async fn persisted_rebalance_requires_worker_activation<S>(pool: Arc<S>, expected_id: &str) -> Result<bool>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load_with_opts(
|
||||
pool,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if persisted.id != expected_id {
|
||||
return Err(Error::other(format!(
|
||||
"rebalance metadata changed before worker activation: expected {expected_id}, found {}",
|
||||
persisted.id
|
||||
)));
|
||||
}
|
||||
Ok(crate::services::rebalance::rebalance_requires_worker_activation(&persisted))
|
||||
}
|
||||
|
||||
pub(super) struct RebalanceRunGuard {
|
||||
_local_guard: tokio::sync::OwnedRwLockReadGuard<()>,
|
||||
persisted_guard: rustfs_lock::NamespaceLockGuard,
|
||||
@@ -577,48 +600,58 @@ impl ECStore {
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
|
||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin. These
|
||||
// shared fences admit a peer without waiting for the run's own entries.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
|
||||
// Classify the durable rebalance record while holding both namespace
|
||||
// fences. A terminal record is a no-op and must not depend on the
|
||||
// notification subsystem having published a fleet proof yet.
|
||||
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
|
||||
let mut worker_fence = acquire_pool_rebalance_worker_locks(pool.clone()).await?;
|
||||
let pool_meta = self
|
||||
.load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation")
|
||||
.load_runtime_pool_meta_under_worker_fence(&mut pool_meta_guard, &worker_fence, "rebalance worker activation")
|
||||
.await?;
|
||||
ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
|
||||
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load_with_opts(
|
||||
pool,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if persisted.id != expected_id {
|
||||
return Err(Error::other(format!(
|
||||
"rebalance metadata changed before worker activation: expected {expected_id}, found {}",
|
||||
persisted.id
|
||||
)));
|
||||
}
|
||||
|
||||
activation_fence.ensure_held()?;
|
||||
if !crate::services::rebalance::rebalance_requires_worker_activation(&persisted) {
|
||||
let needs_activation = persisted_rebalance_requires_worker_activation(pool, expected_id).await?;
|
||||
worker_fence.ensure_held()?;
|
||||
if !needs_activation {
|
||||
return Ok(RebalanceWorkerActivationFence::NotStartedTerminal);
|
||||
}
|
||||
|
||||
// Active worker admission still requires the fail-closed fleet proof.
|
||||
// Attach it immediately before the final fence validation so expiry or
|
||||
// topology changes are checked again at every later commit boundary.
|
||||
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
|
||||
worker_fence.set_fleet_proof(fleet_proof);
|
||||
worker_fence.ensure_held()?;
|
||||
Ok(RebalanceWorkerActivationFence::Ready(Box::new(worker_fence)))
|
||||
}
|
||||
|
||||
pub(super) async fn fence_rebalance_worker_metadata_update<S>(
|
||||
&self,
|
||||
pool: Arc<S>,
|
||||
expected_id: &str,
|
||||
) -> Result<Option<Box<PoolRebalanceActivationFence>>>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
// The admission read fences must already be dropped. Reacquire in the
|
||||
// original order and revalidate every durable boundary across that gap.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker metadata update")?;
|
||||
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
|
||||
let pool_meta = self
|
||||
.load_runtime_pool_meta_under_activation_fence(
|
||||
&mut pool_meta_guard,
|
||||
&activation_fence,
|
||||
"rebalance worker metadata update",
|
||||
)
|
||||
.await?;
|
||||
ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
|
||||
let needs_activation = persisted_rebalance_requires_worker_activation(pool, expected_id).await?;
|
||||
activation_fence.ensure_held()?;
|
||||
if !needs_activation {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
|
||||
activation_fence.set_fleet_proof(fleet_proof);
|
||||
activation_fence.ensure_held()?;
|
||||
|
||||
Ok(RebalanceWorkerActivationFence::Ready(Box::new(activation_fence)))
|
||||
Ok(Some(Box::new(activation_fence)))
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
@@ -1816,6 +1849,107 @@ mod tests {
|
||||
assert_activation_locks_released(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_init_missing_fleet_proof_preserves_metadata_and_rechecks_competing_run() {
|
||||
let previous_id = "completed-before-capability-refresh";
|
||||
let terminal = RebalanceMeta {
|
||||
id: previous_id.to_string(),
|
||||
stopped_at: Some(OffsetDateTime::now_utc()),
|
||||
pool_stats: vec![RebalanceStats::default(); 2],
|
||||
..Default::default()
|
||||
};
|
||||
let (_temp_dirs, store, competing_store) = crate::services::rebalance::test_two_pool_stores(Some(terminal.clone())).await;
|
||||
let disk_stats = vec![
|
||||
DiskStat {
|
||||
total_space: 100,
|
||||
available_space: 0,
|
||||
},
|
||||
DiskStat {
|
||||
total_space: 100,
|
||||
available_space: 100,
|
||||
},
|
||||
];
|
||||
let initial_epoch = store.ctx.data_movement_operation_epoch();
|
||||
let proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
|
||||
set_rebalance_disk_stats_override_for_test(store.id, disk_stats.clone());
|
||||
let err = store
|
||||
.init_rebalance_start(vec!["waiting-request".to_string()])
|
||||
.await
|
||||
.expect_err("initialization must fail closed while the live proof is missing");
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("pool activation requires a live fleet capability proof")
|
||||
);
|
||||
assert!(
|
||||
crate::core::pools::is_pool_activation_fleet_proof_error(&err),
|
||||
"the admin handler retries init only on this error shape: {err}"
|
||||
);
|
||||
assert_eq!(store.ctx.data_movement_operation_epoch(), initial_epoch);
|
||||
{
|
||||
let local = store.rebalance_meta.read().await;
|
||||
let local = local.as_ref().expect("failed initialization must preserve the previous run");
|
||||
assert_eq!(local.id, previous_id);
|
||||
assert_eq!(local.stopped_at, terminal.stopped_at);
|
||||
assert!(local.cancel.is_none(), "failed initialization must not start a worker");
|
||||
}
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load(store.pools[0].clone())
|
||||
.await
|
||||
.expect("failed initialization must leave durable metadata readable");
|
||||
assert_eq!(persisted.id, previous_id);
|
||||
assert_eq!(persisted.stopped_at, terminal.stopped_at);
|
||||
|
||||
drop(proof_guard);
|
||||
set_rebalance_disk_stats_override_for_test(competing_store.id, disk_stats.clone());
|
||||
let winning_id = competing_store
|
||||
.init_rebalance_start(vec!["winning-request".to_string()])
|
||||
.await
|
||||
.expect("a ready request must be able to initialize one run after the failed attempt");
|
||||
assert_ne!(winning_id, previous_id);
|
||||
|
||||
// The waiting request still holds its old local snapshot. Its retry must
|
||||
// recheck durable state under the activation locks before publishing.
|
||||
set_rebalance_disk_stats_override_for_test(store.id, disk_stats);
|
||||
let remote_conflict = store
|
||||
.init_rebalance_start(vec!["waiting-request".to_string()])
|
||||
.await
|
||||
.expect_err("the waiting request must not replace a durable competing run");
|
||||
assert!(
|
||||
matches!(
|
||||
crate::data_movement::data_movement_stage_source(&remote_conflict),
|
||||
Some(Error::RebalanceAlreadyRunning)
|
||||
),
|
||||
"the save-stage error must retain the typed competing-run conflict: {remote_conflict:?}"
|
||||
);
|
||||
let local_conflict = competing_store
|
||||
.init_rebalance_start(vec!["another-request".to_string()])
|
||||
.await
|
||||
.expect_err("a local retry must not replace its active run");
|
||||
assert!(
|
||||
matches!(local_conflict, Error::RebalanceAlreadyRunning),
|
||||
"the local admission check must report the competing-run conflict: {local_conflict:?}"
|
||||
);
|
||||
persisted
|
||||
.load(store.pools[0].clone())
|
||||
.await
|
||||
.expect("the competing run must remain durable after both rejected retries");
|
||||
assert_eq!(persisted.id, winning_id);
|
||||
assert_eq!(persisted.pool_stats[0].buckets, vec!["winning-request".to_string()]);
|
||||
assert_eq!(
|
||||
store
|
||||
.rebalance_meta
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.expect("old local run must survive")
|
||||
.id,
|
||||
previous_id
|
||||
);
|
||||
assert_eq!(store.ctx.data_movement_operation_epoch(), initial_epoch + 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_skips_terminal_metadata_without_fleet_proof() {
|
||||
@@ -1874,6 +2008,128 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_worker_write_upgrade_revalidates_durable_state(change: &str) {
|
||||
let rebalance_id = "rebalance-worker-write-upgrade";
|
||||
let active = RebalanceMeta {
|
||||
id: rebalance_id.to_string(),
|
||||
percent_free_goal: 0.5,
|
||||
pool_stats: vec![
|
||||
RebalanceStats {
|
||||
participating: true,
|
||||
init_capacity: 100,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
RebalanceStats::default(),
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
let (_temp_dirs, store, peer) =
|
||||
crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(active.clone())).await;
|
||||
let arrived = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let worker = Arc::clone(&store);
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
tasks.spawn(
|
||||
super::super::runtime::REBALANCE_WORKER_WRITE_UPGRADE_BARRIER
|
||||
.scope((Arc::clone(&arrived), Arc::clone(&release)), async move {
|
||||
worker.start_rebalance_for_id(rebalance_id).await
|
||||
}),
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), arrived.notified())
|
||||
.await
|
||||
.expect("empty-queue activation must release its read fences before requesting write authority");
|
||||
|
||||
let mut changed = active.clone();
|
||||
match change {
|
||||
"stopped" => {
|
||||
changed.stopped_at = Some(OffsetDateTime::now_utc());
|
||||
changed.pool_stats[0].info.status = RebalStatus::Stopped;
|
||||
}
|
||||
"replaced" => changed.id = "replacement-rebalance-run".to_string(),
|
||||
"decommission" => {
|
||||
let mut pool_meta = peer.pool_meta.read().await.clone();
|
||||
pool_meta.pools[0].decommission = Some(crate::core::pools::PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
});
|
||||
pool_meta
|
||||
.save(peer.pools.clone())
|
||||
.await
|
||||
.expect("a competing pool metadata writer must acquire the released pool fence");
|
||||
}
|
||||
_ => panic!("unknown durable state transition"),
|
||||
}
|
||||
changed
|
||||
.save_with_opts(
|
||||
peer.pools[0].clone(),
|
||||
ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a competing run writer must acquire the released rebalance fence");
|
||||
release.notify_one();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(30), tasks.join_next())
|
||||
.await
|
||||
.expect("activation must finish after the durable transition")
|
||||
.expect("activation task must be tracked")
|
||||
.expect("activation must not panic");
|
||||
match change {
|
||||
"stopped" => result.expect("a durably stopped run must become an activation no-op"),
|
||||
"replaced" => assert!(
|
||||
result
|
||||
.expect_err("a replaced run must reject stale activation")
|
||||
.to_string()
|
||||
.contains("expected"),
|
||||
),
|
||||
"decommission" => assert!(matches!(result, Err(Error::DecommissionAlreadyRunning))),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load(peer.pools[0].clone())
|
||||
.await
|
||||
.expect("the competing durable state must remain readable");
|
||||
assert_eq!(persisted.id, changed.id, "activation must not overwrite the run identity");
|
||||
assert_eq!(persisted.stopped_at, changed.stopped_at, "activation must not overwrite the stop marker");
|
||||
assert_eq!(persisted.pool_stats[0].info.status, changed.pool_stats[0].info.status);
|
||||
assert!(
|
||||
store
|
||||
.rebalance_meta
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.expect("local metadata must remain present")
|
||||
.cancel
|
||||
.is_none(),
|
||||
"rejected activation must not install a worker token",
|
||||
);
|
||||
assert_activation_locks_released(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_write_upgrade_rechecks_stop() {
|
||||
assert_worker_write_upgrade_revalidates_durable_state("stopped").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_write_upgrade_rechecks_run_identity() {
|
||||
assert_worker_write_upgrade_revalidates_durable_state("replaced").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_write_upgrade_rechecks_decommission() {
|
||||
assert_worker_write_upgrade_revalidates_durable_state("decommission").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() {
|
||||
|
||||
@@ -1440,7 +1440,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn real_rebalance_entry_progresses_while_peer_activation_waits_for_run_fence() {
|
||||
async fn real_rebalance_peer_activates_while_entry_retains_run_fence() {
|
||||
const REBALANCE_ID: &str = "rebalance-peer-activation-lock-order";
|
||||
let (_temp_dirs, store, peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(
|
||||
active_rebalance_meta(REBALANCE_ID),
|
||||
@@ -1456,7 +1456,7 @@ mod tests {
|
||||
let bucket = crate::disk::RUSTFS_META_BUCKET;
|
||||
let object = "rebalance-peer-activation-object";
|
||||
let version_id = uuid::Uuid::new_v4();
|
||||
let payload = b"entry must drain before peer activation takes the pool fence".repeat(1024);
|
||||
let payload = b"peer activation must not wait for this entry to drain".repeat(1024);
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
let opts = ObjectOptions {
|
||||
@@ -1516,7 +1516,12 @@ mod tests {
|
||||
);
|
||||
tokio::time::timeout(StdDuration::from_secs(30), attempted.notified())
|
||||
.await
|
||||
.expect("peer activation must attempt the persisted rebalance write fence");
|
||||
.expect("peer activation must attempt the persisted rebalance fence");
|
||||
tokio::time::timeout(StdDuration::from_secs(30), activation_result)
|
||||
.await
|
||||
.expect("peer activation must finish while the entry remains paused")
|
||||
.expect("peer activation result should be sent")
|
||||
.expect("joining the same run must not require its active entry to release the run fence");
|
||||
release.notify_one();
|
||||
|
||||
tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
@@ -1531,10 +1536,6 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect("entry and peer activation must both make progress");
|
||||
activation_result
|
||||
.await
|
||||
.expect("peer activation result should be sent")
|
||||
.expect("peer activation must not time out behind the entry it blocks");
|
||||
|
||||
let mut reader = target_set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
|
||||
@@ -71,6 +71,31 @@ impl From<u8> for RebalStatus {
|
||||
}
|
||||
|
||||
impl RebalanceMeta {
|
||||
/// A historical run has no authority over the current deployment's pool
|
||||
/// indices. Stop timestamps alone do not prove that its workers stopped.
|
||||
pub(crate) fn has_terminal_topology(&self) -> bool {
|
||||
!self.id.is_empty()
|
||||
&& !self.pool_stats.is_empty()
|
||||
&& self.pool_stats.iter().all(|pool| {
|
||||
if pool.info.stopping
|
||||
|| !matches!(pool.info.status, RebalStatus::Completed | RebalStatus::Stopped | RebalStatus::Failed)
|
||||
&& (pool.participating || pool.info.status != RebalStatus::None)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let Some(message) = pool.info.last_error.as_deref() else {
|
||||
return true;
|
||||
};
|
||||
if !is_rebalance_stop_propagation_error(Some(message)) {
|
||||
return true;
|
||||
}
|
||||
decode_rebalance_stop_propagation_record(message).is_some_and(|record| {
|
||||
!record.has_failures()
|
||||
&& (record.stop_attempt_at.or(self.stopped_at).is_none() || record.terminal_reload_attempt_at.is_some())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
@@ -27,6 +27,11 @@ use tokio::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(super) static REBALANCE_WORKER_WRITE_UPGRADE_BARRIER: (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>);
|
||||
}
|
||||
|
||||
pub(super) fn should_fail_repeated_rebalance_bucket_defer(
|
||||
deferred_buckets: &mut HashSet<String>,
|
||||
bucket: &str,
|
||||
@@ -213,6 +218,22 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if must_persist {
|
||||
// Shared admission grants no write authority. Release it before
|
||||
// taking the exclusive metadata fences, then recheck stop/run/pool
|
||||
// state before publishing the locally staged terminal candidate.
|
||||
drop(activation_fence);
|
||||
#[cfg(test)]
|
||||
if let Ok((arrived, release)) = REBALANCE_WORKER_WRITE_UPGRADE_BARRIER.try_with(Clone::clone) {
|
||||
arrived.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
let Some(activation_fence) = self
|
||||
.fence_rebalance_worker_metadata_update(pool.clone(), expected_id.as_ref())
|
||||
.await?
|
||||
else {
|
||||
cancel_tx.cancel();
|
||||
return Ok(false);
|
||||
};
|
||||
let save_result = resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_under_activation_fence(
|
||||
pool,
|
||||
@@ -242,13 +263,14 @@ impl ECStore {
|
||||
cancel_tx.cancel();
|
||||
return Err(err);
|
||||
}
|
||||
} else {
|
||||
if let Err(err) = activation_fence.ensure_held() {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
|
||||
return Err(err);
|
||||
}
|
||||
drop(activation_fence);
|
||||
}
|
||||
if !must_persist && let Err(err) = activation_fence.ensure_held() {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
|
||||
return Err(err);
|
||||
}
|
||||
drop(activation_fence);
|
||||
|
||||
if activation_outcome != RebalanceLocalActivationOutcome::Started {
|
||||
return Ok(must_persist);
|
||||
|
||||
@@ -2270,7 +2270,7 @@ pub(crate) fn tier_destination_id_from_metadata(metadata: &HashMap<String, Strin
|
||||
}
|
||||
return Ok(None);
|
||||
};
|
||||
if encoded.len() != 64 {
|
||||
if encoded.len() != 64 || !encoded.is_ascii() {
|
||||
return Err(io::Error::other("transition tier backend identity has an invalid length"));
|
||||
}
|
||||
let mut identity = [0_u8; 32];
|
||||
@@ -4715,6 +4715,23 @@ impl TierConfigMgr {
|
||||
}
|
||||
|
||||
async fn reload_handle_with<S>(handle: &Arc<RwLock<Self>>, api: Arc<S>) -> io::Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO
|
||||
+ EcstoreObjectOperations
|
||||
+ NamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
+ ListOperations<Error = Error, ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>>,
|
||||
{
|
||||
// Periodic and committed-mutation refreshes call this shared entry directly.
|
||||
let reload = Self::reload_handle_inner(handle, api);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let reload = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::TierConfigReload,
|
||||
reload,
|
||||
);
|
||||
reload.await
|
||||
}
|
||||
|
||||
async fn reload_handle_inner<S>(handle: &Arc<RwLock<Self>>, api: Arc<S>) -> io::Result<()>
|
||||
where
|
||||
S: EcstoreObjectIO
|
||||
+ EcstoreObjectOperations
|
||||
@@ -12533,6 +12550,51 @@ mod tests {
|
||||
assert!(reloaded.iter().any(|intent| intent.mutation_id == prepared_id));
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn diagnostic_reload_shared_entry_preserves_source_and_api() {
|
||||
use crate::api::diagnostic::{ReadVersionEntrySource, current_read_version_entry_source, with_read_version_entry};
|
||||
use rustfs_io_metrics::s3_http_metrics::{
|
||||
S3HttpRequestGuard, current_s3_http_operation_context, with_isolated_s3_http_operation_context,
|
||||
};
|
||||
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
empty_mgr()
|
||||
.save_tiering_config_if_current(store.clone(), None)
|
||||
.await
|
||||
.expect("empty tier config fixture should persist");
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let get = request.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
|
||||
current_s3_http_operation_context().expect("GetObject context should be captured")
|
||||
});
|
||||
|
||||
for (earlier, api) in [(None, None), (Some(ReadVersionEntrySource::ConfigRead), Some(get))] {
|
||||
store.reload_contexts.lock().expect("context recorder").clear();
|
||||
let handle = TierConfigMgr::new();
|
||||
with_isolated_s3_http_operation_context(api, async {
|
||||
let reload = TierConfigMgr::reload_handle_with(&handle, store.clone());
|
||||
if let Some(source) = earlier {
|
||||
with_read_version_entry(source, reload).await
|
||||
} else {
|
||||
reload.await
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("shared reload entry should load config and scan intents");
|
||||
|
||||
let observed = store.reload_contexts.lock().expect("context recorder");
|
||||
assert!(observed.iter().any(|(boundary, _, _)| *boundary == "reader"));
|
||||
assert!(observed.iter().any(|(boundary, _, _)| *boundary == "list"));
|
||||
let expected = earlier.unwrap_or(ReadVersionEntrySource::TierConfigReload);
|
||||
assert!(observed.iter().all(|(_, source, operation)| {
|
||||
*source == Some(expected) && *operation == api.map(|context| context.operation_name())
|
||||
}));
|
||||
assert_eq!(current_read_version_entry_source(), None);
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reload_handle_keeps_committed_block_until_publish_swaps_generation() {
|
||||
let store = Arc::new(CasConfigStore::default());
|
||||
@@ -14725,8 +14787,17 @@ mod tests {
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
type ReloadDiagnosticObservation = (
|
||||
&'static str,
|
||||
Option<rustfs_io_metrics::internode_metrics::ReadVersionEntrySource>,
|
||||
Option<&'static str>,
|
||||
);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CasConfigStore {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
reload_contexts: Mutex<Vec<ReloadDiagnosticObservation>>,
|
||||
objects: tokio::sync::Mutex<HashMap<String, (Vec<u8>, String)>>,
|
||||
legacy_state: tokio::sync::Mutex<Option<Vec<u8>>>,
|
||||
if_match_race_rewrite: tokio::sync::Mutex<std::collections::VecDeque<(String, Vec<u8>)>>,
|
||||
@@ -14750,6 +14821,8 @@ mod tests {
|
||||
impl Default for CasConfigStore {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
reload_contexts: Mutex::new(Vec::new()),
|
||||
objects: tokio::sync::Mutex::new(HashMap::new()),
|
||||
legacy_state: tokio::sync::Mutex::new(None),
|
||||
if_match_race_rewrite: tokio::sync::Mutex::new(std::collections::VecDeque::new()),
|
||||
@@ -14904,6 +14977,12 @@ mod tests {
|
||||
_headers: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
self.reload_contexts.lock().expect("context recorder").push((
|
||||
"reader",
|
||||
crate::api::diagnostic::current_read_version_entry_source(),
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
));
|
||||
let legacy_config_path = tier_config_path(TIER_CONFIG_LEGACY_FILE);
|
||||
let (data, etag) = if object == legacy_config_path {
|
||||
let state = self.legacy_state.lock().await;
|
||||
@@ -15259,6 +15338,12 @@ mod tests {
|
||||
_start_after: Option<String>,
|
||||
_incl_deleted: bool,
|
||||
) -> Result<Self::ListObjectsV2Info> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
self.reload_contexts.lock().expect("context recorder").push((
|
||||
"list",
|
||||
crate::api::diagnostic::current_read_version_entry_source(),
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
));
|
||||
if matches!(
|
||||
prefix,
|
||||
crate::services::tier::tier_mutation_intent::TIER_MUTATION_INTENT_RECORD_PREFIX
|
||||
|
||||
@@ -6205,12 +6205,10 @@ impl SetDisks {
|
||||
/// * The set of referenced data dirs is the UNION of `get_data_dirs()` across
|
||||
/// every online disk's `xl.meta`, so a dir named by *any* replica is kept.
|
||||
/// * If a disk holds the object directory but its `xl.meta` is missing or
|
||||
/// unparsable, the object is treated as degraded and unmarked data dirs are
|
||||
/// never removed. A data dir carrying a committed delete-transaction marker
|
||||
/// remains reclaimable after a downgrade/re-upgrade cleanup interruption.
|
||||
/// * Only subdirectories whose names parse as a UUID are ever considered;
|
||||
/// removal is non-recursive-safe via a recursive delete of the full stray
|
||||
/// data-dir path only.
|
||||
/// unparsable, leave its data to quorum-aware recovery and healing.
|
||||
/// * Any outstanding delete transaction defers all candidates to recovery.
|
||||
/// * UUID-shaped S3 prefixes are protected: only regular part files and native
|
||||
/// sidecars qualify. The final mutation repeats this check and never recurses.
|
||||
///
|
||||
/// Returns the number of stray data directories removed across the set. This
|
||||
/// is best-effort maintenance: individual delete failures are logged and
|
||||
@@ -6225,13 +6223,16 @@ impl SetDisks {
|
||||
|
||||
async fn reclaim_orphan_data_dirs_inner(&self, bucket: &str, object: &str, dry_run: bool) -> disk::error::Result<usize> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() != self.set_drive_count || disks.iter().any(Option::is_none) {
|
||||
// An unavailable participant can still reference an apparent orphan.
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
// Phase 1 (read-only): build the referenced-data-dir union and record the
|
||||
// physical UUID subdirectories present on each disk. Abort on any degraded
|
||||
// copy so a healable object is never stripped of a referenced data dir.
|
||||
let mut referenced: HashSet<Uuid> = HashSet::new();
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<(Uuid, bool)>)> = Vec::new();
|
||||
let mut healthy_metas = 0usize;
|
||||
let mut per_disk_dirs: Vec<(usize, Vec<Uuid>)> = Vec::new();
|
||||
|
||||
for (i, disk) in disks.iter().enumerate() {
|
||||
let Some(disk) = disk else { continue };
|
||||
@@ -6266,22 +6267,6 @@ impl SetDisks {
|
||||
// to the orphan-dir / dangling-object heal paths.
|
||||
continue;
|
||||
}
|
||||
let mut committed = Vec::with_capacity(physical.len());
|
||||
for dir in physical {
|
||||
let data_dir = format!("{object}/{dir}");
|
||||
let committed_delete = disk.list_dir("", bucket, &data_dir, 0).await.is_ok_and(|entries| {
|
||||
entries.iter().any(|entry| {
|
||||
entry
|
||||
.strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
|
||||
.is_some_and(|transaction| Uuid::parse_str(transaction).is_ok())
|
||||
})
|
||||
});
|
||||
committed.push((dir, committed_delete));
|
||||
}
|
||||
if committed.iter().all(|(_, committed_delete)| *committed_delete) {
|
||||
per_disk_dirs.push((i, committed));
|
||||
continue;
|
||||
}
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
bucket, object,
|
||||
@@ -6316,21 +6301,39 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
healthy_metas += 1;
|
||||
if !physical.is_empty() {
|
||||
per_disk_dirs.push((i, physical.into_iter().map(|dir| (dir, false)).collect()));
|
||||
per_disk_dirs.push((i, physical));
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: delete every physical data dir not referenced by the union.
|
||||
if !per_disk_dirs
|
||||
.iter()
|
||||
.any(|(_, dirs)| dirs.iter().any(|dir| !referenced.contains(dir)))
|
||||
{
|
||||
return Ok(0);
|
||||
}
|
||||
// A neighboring journal can still need an unreferenced payload for
|
||||
// rollback. Objects whose physical dirs are all referenced pay no
|
||||
// recovery-discovery overhead.
|
||||
for disk in disks.iter().flatten() {
|
||||
match disk.has_delete_transaction_evidence(bucket, object).await {
|
||||
Ok(false) => {}
|
||||
_ => return Ok(0),
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: classify candidates consistently for dry-run and reclamation.
|
||||
let mut removed = 0usize;
|
||||
for (i, physical) in per_disk_dirs {
|
||||
let Some(disk) = disks[i].as_ref() else { continue };
|
||||
for (dir, committed_delete) in physical {
|
||||
if referenced.contains(&dir) || (healthy_metas == 0 && !committed_delete) {
|
||||
for dir in physical {
|
||||
if referenced.contains(&dir) {
|
||||
continue;
|
||||
}
|
||||
let stray = format!("{object}/{dir}");
|
||||
if !disk.is_plain_orphan_data_dir(bucket, &stray).await.unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
if dry_run {
|
||||
removed += 1;
|
||||
debug!(
|
||||
@@ -6346,18 +6349,18 @@ impl SetDisks {
|
||||
continue;
|
||||
}
|
||||
match disk
|
||||
.delete(
|
||||
.delete_data_dir(
|
||||
bucket,
|
||||
&stray,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
orphan_data_dir: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
Ok(DataDirDeleteStatus::Deleted) => {
|
||||
removed += 1;
|
||||
info!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
@@ -6365,7 +6368,8 @@ impl SetDisks {
|
||||
"reclaim_orphan_data_dirs: removed orphaned data directory"
|
||||
);
|
||||
}
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
|
||||
Ok(DataDirDeleteStatus::Deferred)
|
||||
| Err(DiskError::FileNotFound | DiskError::VolumeNotFound | DiskError::DiskOngoingReq) => {}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "rustfs_ecstore::set_disk",
|
||||
@@ -6413,7 +6417,18 @@ impl SetDisks {
|
||||
// - If no_lock is false, we must have obtained the lock out side of this function
|
||||
// - If no_lock is true, we should not obtain locks
|
||||
lookup_opts.no_lock = true;
|
||||
let oi = self.get_object_info(bucket, object, &lookup_opts).await;
|
||||
let lookup = self.get_object_info(bucket, object, &lookup_opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let lookup = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::WritePreconditionRead,
|
||||
async {
|
||||
crate::cluster::rpc::runtime_sources::observe_read_version_context_boundary(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::WritePreconditionLookup,
|
||||
);
|
||||
lookup.await
|
||||
},
|
||||
);
|
||||
let oi = lookup.await;
|
||||
|
||||
match oi {
|
||||
Ok(oi) => {
|
||||
@@ -6776,6 +6791,12 @@ pub(crate) mod disk_call_counters {
|
||||
/// Registry key: (object, kind, disk_index).
|
||||
type CountKey = (String, String, usize);
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
type ReadVersionContext = (
|
||||
Option<&'static str>,
|
||||
Option<rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller>,
|
||||
);
|
||||
|
||||
#[derive(Default)]
|
||||
struct Registry {
|
||||
/// Object names with an active observing scope. Only these accumulate,
|
||||
@@ -6783,6 +6804,8 @@ pub(crate) mod disk_call_counters {
|
||||
observed: HashSet<String>,
|
||||
/// (object, kind, disk_index) -> call count.
|
||||
counts: HashMap<CountKey, u64>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
read_version_contexts: HashMap<(String, usize), Vec<ReadVersionContext>>,
|
||||
}
|
||||
|
||||
fn registry() -> &'static Mutex<Registry> {
|
||||
@@ -6800,6 +6823,17 @@ pub(crate) mod disk_call_counters {
|
||||
*reg.counts
|
||||
.entry((object.to_string(), kind.to_string(), disk_index))
|
||||
.or_insert(0) += 1;
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
if kind == KIND_READ_VERSION {
|
||||
let context = (
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
crate::cluster::rpc::runtime_sources::current_read_version_request_caller(),
|
||||
);
|
||||
reg.read_version_contexts
|
||||
.entry((object.to_string(), disk_index))
|
||||
.or_default()
|
||||
.push(context);
|
||||
}
|
||||
}
|
||||
|
||||
/// Matches shard paths to the observed object's namespace path. This also
|
||||
@@ -6842,6 +6876,15 @@ pub(crate) mod disk_call_counters {
|
||||
}
|
||||
|
||||
impl CallCounterScope {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub fn read_version_contexts_for_disk(&self, disk_index: usize) -> Vec<ReadVersionContext> {
|
||||
let reg = registry().lock().expect("disk call-counter registry poisoned");
|
||||
reg.read_version_contexts
|
||||
.get(&(self.object.clone(), disk_index))
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Total number of `kind` calls across all disks for the observed object.
|
||||
pub fn total(&self, kind: &str) -> u64 {
|
||||
let reg = registry().lock().expect("disk call-counter registry poisoned");
|
||||
@@ -6867,6 +6910,8 @@ pub(crate) mod disk_call_counters {
|
||||
let mut reg = registry().lock().expect("disk call-counter registry poisoned");
|
||||
reg.observed.remove(&self.object);
|
||||
reg.counts.retain(|(obj, _, _), _| obj != &self.object);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
reg.read_version_contexts.retain(|(obj, _), _| obj != &self.object);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8008,6 +8053,290 @@ mod tests {
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[test]
|
||||
fn diagnostic_coalescer_restores_each_item_after_delayed_and_threshold_flush() {
|
||||
use crate::cluster::rpc::runtime_sources::{
|
||||
ReadVersionDiagnosticContext, with_optional_read_version_request_caller, with_read_version_diagnostic_context,
|
||||
with_read_version_entry,
|
||||
};
|
||||
use crate::set_disk::core::metadata_read::enqueue_coalesced_read_version;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use rustfs_io_metrics::internode_metrics::{
|
||||
ReadVersionContextBoundary, ReadVersionEntrySource, ReadVersionRequestCaller,
|
||||
};
|
||||
use rustfs_io_metrics::s3_http_metrics::{
|
||||
S3HttpRequestGuard, current_s3_http_operation_context, with_isolated_s3_http_operation_context,
|
||||
};
|
||||
let operation = |op| {
|
||||
let mut guard = S3HttpRequestGuard::new("GET");
|
||||
guard.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(op);
|
||||
current_s3_http_operation_context().expect("operation should be captured")
|
||||
})
|
||||
};
|
||||
let get = operation(rustfs_s3_ops::S3Operation::GetObject);
|
||||
let list = operation(rustfs_s3_ops::S3Operation::ListObjectsV2);
|
||||
let foreign = operation(rustfs_s3_ops::S3Operation::DeleteObject);
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
|
||||
for (items, delay) in [(4, "2000"), (crate::disk::BATCH_READ_VERSION_MAX_ITEMS, "10000")] {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshots = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
runtime.block_on(async {
|
||||
let bucket = "diagnostic-coalescer-context";
|
||||
let (_dirs, disks) = call_counter_local_disks(bucket, 1).await;
|
||||
let disk = disks[0].clone().expect("test disk should exist");
|
||||
for case in 0..4 {
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, &format!("context-{case}"), None).await;
|
||||
}
|
||||
let mut contexts = Vec::new();
|
||||
for (api, caller, source) in [
|
||||
(Some(get), Some(ReadVersionRequestCaller::Object), ReadVersionEntrySource::ObjectRead),
|
||||
(None, None, ReadVersionEntrySource::ConfigRead),
|
||||
(
|
||||
Some(list),
|
||||
Some(ReadVersionRequestCaller::ListObjects),
|
||||
ReadVersionEntrySource::ListObjects,
|
||||
),
|
||||
(None, Some(ReadVersionRequestCaller::Heal), ReadVersionEntrySource::LifecycleRecovery),
|
||||
] {
|
||||
contexts.push(
|
||||
with_isolated_s3_http_operation_context(
|
||||
api,
|
||||
with_optional_read_version_request_caller(
|
||||
caller,
|
||||
with_read_version_entry(source, async { ReadVersionDiagnosticContext::capture() }),
|
||||
),
|
||||
)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
let opts = ReadOptions::default();
|
||||
let foreign_context = with_isolated_s3_http_operation_context(
|
||||
Some(foreign),
|
||||
with_optional_read_version_request_caller(
|
||||
Some(ReadVersionRequestCaller::Multipart),
|
||||
with_read_version_entry(ReadVersionEntrySource::ObjectRead, async {
|
||||
ReadVersionDiagnosticContext::capture()
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
temp_env::async_with_vars([("RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS", Some(delay))], async {
|
||||
with_read_version_diagnostic_context(foreign_context, ReadVersionContextBoundary::Entry, async {
|
||||
let reads = (0..items).map(|index| {
|
||||
let case = index % contexts.len();
|
||||
let item = BatchReadVersionItem {
|
||||
org_volume: String::new(),
|
||||
volume: bucket.to_owned(),
|
||||
path: format!("context-{case}"),
|
||||
version_id: String::new(),
|
||||
};
|
||||
with_read_version_diagnostic_context(
|
||||
contexts[case],
|
||||
ReadVersionContextBoundary::EarlyStop,
|
||||
enqueue_coalesced_read_version(disk.clone(), item, &opts),
|
||||
)
|
||||
});
|
||||
let results = futures::future::join_all(reads).await;
|
||||
for (index, result) in results.into_iter().enumerate() {
|
||||
assert_eq!(result.expect("fallback read should succeed").name, format!("context-{}", index % 4));
|
||||
}
|
||||
assert_eq!(
|
||||
current_s3_http_operation_context(),
|
||||
Some(foreign),
|
||||
"flush must restore its caller's ambient scope"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
})
|
||||
.await;
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
})
|
||||
});
|
||||
let rows = snapshots.snapshot().into_vec();
|
||||
let item_label = items.to_string();
|
||||
assert!(
|
||||
rows.iter().any(|(key, _, _, value)| {
|
||||
let labels = key
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key(), label.value()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
key.key().name() == "rustfs_get_metadata_read_version_coalescer_total"
|
||||
&& labels.get("event") == Some(&"attempted_batch")
|
||||
&& labels.get("item_count") == Some(&item_label.as_str())
|
||||
&& matches!(value, DebugValue::Counter(1))
|
||||
}),
|
||||
"all requests should flush in exactly one batch"
|
||||
);
|
||||
for (source, api, caller) in [
|
||||
("object_read", "s3:GetObject", "object"),
|
||||
("config_read", "unattributed", "unattributed"),
|
||||
("list_objects", "s3:ListObjectsV2", "list_objects"),
|
||||
("lifecycle_recovery", "unattributed", "heal"),
|
||||
] {
|
||||
assert!(rows.iter().any(|(key, _, _, value)| {
|
||||
let labels = key.key().labels().map(|label| (label.key(), label.value())).collect::<std::collections::HashMap<_, _>>();
|
||||
key.key().name() == "rustfs_storage_diagnostic_context_transitions_total"
|
||||
&& labels.get("boundary") == Some(&"unary_fallback")
|
||||
&& labels.get("entry_source") == Some(&source)
|
||||
&& labels.get("parent_api_operation") == Some(&api)
|
||||
&& labels.get("child_api_operation") == Some(&api)
|
||||
&& labels.get("parent_caller") == Some(&caller)
|
||||
&& labels.get("child_caller") == Some(&caller)
|
||||
&& matches!(value, DebugValue::Counter(count) if *count == u64::try_from(items / 4).expect("small item count"))
|
||||
}), "each item must retain its own API, caller, and entry source");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn metadata_full_wait_preserves_external_request_diagnostic_context() {
|
||||
use crate::cluster::rpc::runtime_sources::with_read_version_request_caller;
|
||||
use rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller;
|
||||
use rustfs_io_metrics::s3_http_metrics::{
|
||||
S3HttpRequestGuard, current_s3_http_operation_context, with_s3_http_operation_context,
|
||||
};
|
||||
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "full-wait-external-context-bucket";
|
||||
let object = "full-wait-external-context-object";
|
||||
let (_dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, object, None).await;
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let operation = request
|
||||
.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
|
||||
current_s3_http_operation_context()
|
||||
})
|
||||
.expect("external request operation should be captured");
|
||||
let (metadata, errors, _) = with_s3_http_operation_context(
|
||||
Some(operation),
|
||||
with_read_version_request_caller(
|
||||
ReadVersionRequestCaller::Object,
|
||||
SetDisks::read_all_fileinfo_observed(&disks, "", bucket, object, "", false, false, false, false, 2),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("full-wait metadata read should complete");
|
||||
|
||||
assert_eq!(metadata.iter().filter(|info| info.name == object).count(), DISKS);
|
||||
assert!(errors.iter().all(Option::is_none));
|
||||
for disk_index in 0..DISKS {
|
||||
assert_eq!(
|
||||
calls.read_version_contexts_for_disk(disk_index),
|
||||
vec![(Some("s3:GetObject"), Some(ReadVersionRequestCaller::Object))],
|
||||
"every full-wait child must retain the external request context"
|
||||
);
|
||||
}
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn metadata_full_wait_keeps_internal_reads_without_s3_context() {
|
||||
use crate::cluster::rpc::runtime_sources::with_optional_read_version_request_caller;
|
||||
use rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller;
|
||||
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "full-wait-internal-context-bucket";
|
||||
let (_dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
for (case, caller) in [None, Some(ReadVersionRequestCaller::Heal)].into_iter().enumerate() {
|
||||
let object = format!("full-wait-internal-context-object-{case}");
|
||||
install_metadata_fanout_fileinfo(&disks, bucket, &object, None).await;
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let (metadata, errors, _) = with_optional_read_version_request_caller(
|
||||
caller,
|
||||
SetDisks::read_all_fileinfo_observed(&disks, "", bucket, &object, "", false, false, false, false, 2),
|
||||
)
|
||||
.await
|
||||
.expect("internal full-wait metadata read should complete");
|
||||
|
||||
assert_eq!(metadata.iter().filter(|info| info.name == object).count(), DISKS);
|
||||
assert!(errors.iter().all(Option::is_none));
|
||||
for disk_index in 0..DISKS {
|
||||
assert_eq!(
|
||||
calls.read_version_contexts_for_disk(disk_index),
|
||||
vec![(None, caller)],
|
||||
"internal reads must retain their caller without acquiring an external request context"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn metadata_early_stop_preserves_external_and_internal_diagnostic_context() {
|
||||
use crate::cluster::rpc::runtime_sources::with_optional_read_version_request_caller;
|
||||
use rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller;
|
||||
use rustfs_io_metrics::s3_http_metrics::{
|
||||
S3HttpRequestGuard, current_s3_http_operation_context, with_s3_http_operation_context,
|
||||
};
|
||||
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "early-stop-diagnostic-context-bucket";
|
||||
let (_dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let operation = request
|
||||
.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::GetObject);
|
||||
current_s3_http_operation_context()
|
||||
})
|
||||
.expect("external request operation should be captured");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
for (case, api, caller) in [
|
||||
(0, Some(operation), Some(ReadVersionRequestCaller::Object)),
|
||||
(1, None, None),
|
||||
(2, None, Some(ReadVersionRequestCaller::Heal)),
|
||||
] {
|
||||
let object = object_with_initial_data_shards(bucket, &format!("early-stop-context-object-{case}"), 2, 3);
|
||||
install_inline_metadata_fanout_fileinfo(&disks, bucket, &object, b"context probe", |_| {}).await;
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let (metadata, errors, _) = with_s3_http_operation_context(
|
||||
api,
|
||||
with_optional_read_version_request_caller(
|
||||
caller,
|
||||
SetDisks::read_all_fileinfo_observed(&disks, "", bucket, &object, "", true, false, false, true, 2),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("early-stop metadata read should complete");
|
||||
|
||||
assert!(metadata.iter().filter(|info| info.name == object).count() >= 2);
|
||||
assert!(errors.iter().all(Option::is_none));
|
||||
assert!(calls.total(disk_call_counters::KIND_READ_VERSION) >= 2);
|
||||
let expected_api = api.map(|context| context.operation_name());
|
||||
for disk_index in 0..DISKS {
|
||||
for context in calls.read_version_contexts_for_disk(disk_index) {
|
||||
assert_eq!(
|
||||
context,
|
||||
(expected_api, caller),
|
||||
"every polled early-stop child must retain its parent's diagnostic context"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_read_version_response_mapping_preserves_index_and_errors() {
|
||||
let expected_items = vec![
|
||||
|
||||
@@ -171,6 +171,23 @@ fn read_version_coalescing_delay() -> Duration {
|
||||
struct CoalescedReadVersionRequest {
|
||||
item: BatchReadVersionItem,
|
||||
tx: oneshot::Sender<disk::error::Result<FileInfo>>,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
context: crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext,
|
||||
}
|
||||
|
||||
impl CoalescedReadVersionRequest {
|
||||
fn new(item: BatchReadVersionItem, tx: oneshot::Sender<disk::error::Result<FileInfo>>) -> Self {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture();
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
context.observe(rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::CoalescerEnqueue);
|
||||
Self {
|
||||
item,
|
||||
tx,
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
context,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
@@ -246,26 +263,49 @@ async fn read_version_via_coalescer(
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
if !allow_coalescing || !read_version_coalescing_enabled() {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_request_caller_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::MetadataRead,
|
||||
disk.read_version(org_bucket, bucket, object, version_id, opts),
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
|
||||
}
|
||||
if !matches!(disk.as_ref(), Disk::Remote(_)) {
|
||||
record_read_version_coalescer_event("bypass_non_remote", 1);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_request_caller_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::MetadataRead,
|
||||
disk.read_version(org_bucket, bucket, object, version_id, opts),
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
return disk.read_version(org_bucket, bucket, object, version_id, opts).await;
|
||||
}
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let item = BatchReadVersionItem {
|
||||
org_volume: org_bucket.to_string(),
|
||||
volume: bucket.to_string(),
|
||||
path: object.to_string(),
|
||||
version_id: version_id.to_string(),
|
||||
};
|
||||
enqueue_coalesced_read_version(disk, item, opts).await
|
||||
}
|
||||
|
||||
pub(super) async fn enqueue_coalesced_read_version(
|
||||
disk: DiskStore,
|
||||
item: BatchReadVersionItem,
|
||||
opts: &ReadOptions,
|
||||
) -> disk::error::Result<FileInfo> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let lane_key = ReadVersionCoalescerKey::new(&disk, opts);
|
||||
let request = CoalescedReadVersionRequest::new(item, tx);
|
||||
let pending = {
|
||||
let mut coalescer = read_version_coalescer().lock().await;
|
||||
let lane = coalescer.lanes.entry(lane_key).or_default();
|
||||
let schedule_delayed_flush = lane.is_empty();
|
||||
lane.push(CoalescedReadVersionRequest { item, tx });
|
||||
lane.push(request);
|
||||
if lane.len() >= BATCH_READ_VERSION_MAX_ITEMS {
|
||||
coalescer.lanes.remove(&lane_key)
|
||||
} else if schedule_delayed_flush {
|
||||
@@ -326,20 +366,24 @@ async fn flush_read_version_coalescer_pending(
|
||||
|
||||
let mut senders = Vec::with_capacity(pending.len());
|
||||
let mut items = Vec::with_capacity(pending.len());
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let mut contexts = Vec::with_capacity(pending.len());
|
||||
for request in pending {
|
||||
senders.push(request.tx);
|
||||
items.push(request.item);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
contexts.push(request.context);
|
||||
}
|
||||
|
||||
let expected_items = items.iter().map(ExpectedBatchReadVersionItem::from).collect::<Vec<_>>();
|
||||
record_read_version_coalescer_event("attempted_batch", items.len());
|
||||
let result =
|
||||
match tokio::time::timeout(get_drive_metadata_timeout(), disk.batch_read_version(BatchReadVersionReq { items, opts }))
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
};
|
||||
let batch = disk.batch_read_version(BatchReadVersionReq { items, opts });
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let batch = crate::cluster::rpc::runtime_sources::with_read_version_batch_contexts(contexts, batch);
|
||||
let result = match tokio::time::timeout(get_drive_metadata_timeout(), batch).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
};
|
||||
match result {
|
||||
Ok(responses) => {
|
||||
let map_started = batch_read_version_stage_timer();
|
||||
@@ -1016,6 +1060,10 @@ impl SetDisks {
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let diagnostic_context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::MetadataRead,
|
||||
);
|
||||
let futures = disks.iter().enumerate().map(|(disk_index, disk)| {
|
||||
let disk = disk.clone();
|
||||
let task_opts = opts;
|
||||
@@ -1024,21 +1072,34 @@ impl SetDisks {
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let slowtail_fault = slowtail_fault.clone();
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let diagnostic_context = diagnostic_context;
|
||||
AbortOnDropJoinHandle(tokio::spawn(async move {
|
||||
let response_start = observe.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, disk_index);
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||
Self::record_metadata_slowtail_fault(&object, disk_index);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
let read = async move {
|
||||
let response_start = observe.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, disk_index);
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(disk_index)) {
|
||||
Self::record_metadata_slowtail_fault(&object, disk_index);
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
let elapsed = response_start.map(|start| start.elapsed());
|
||||
(result, elapsed)
|
||||
};
|
||||
let elapsed = response_start.map(|start| start.elapsed());
|
||||
(result, elapsed)
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
diagnostic_context,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::FullWait,
|
||||
read,
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
read.await
|
||||
}))
|
||||
});
|
||||
|
||||
@@ -1134,6 +1195,10 @@ impl SetDisks {
|
||||
let mut non_inline_candidate_eligible = None;
|
||||
let mut single_pending_hedge_deadline = None;
|
||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let diagnostic_context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::MetadataRead,
|
||||
);
|
||||
let spawn_read_version = |join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>,
|
||||
task_ids: &mut [Option<tokio::task::Id>],
|
||||
index: usize,
|
||||
@@ -1144,27 +1209,44 @@ impl SetDisks {
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let slowtail_fault = slowtail_fault.clone();
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let diagnostic_context = diagnostic_context;
|
||||
let task = join_set.spawn(async move {
|
||||
let response_start = Instant::now();
|
||||
let result = if let Some(disk) = disk {
|
||||
#[cfg(test)]
|
||||
let _fanout_task_guard = rename_fanout_barrier::task_guard(&object);
|
||||
Self::record_read_version_call(&object, index);
|
||||
#[cfg(test)]
|
||||
Self::read_version_fanout_barrier(&object, index).await;
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||
Self::record_metadata_slowtail_fault(&object, index);
|
||||
let read = async move {
|
||||
let response_start = Instant::now();
|
||||
let result = if let Some(disk) = disk {
|
||||
#[cfg(test)]
|
||||
rename_fanout_barrier::checkpoint(&object, index, rename_fanout_barrier::PHASE_METADATA_SLOWTAIL_FAULT)
|
||||
let _fanout_task_guard = rename_fanout_barrier::task_guard(&object);
|
||||
Self::record_read_version_call(&object, index);
|
||||
#[cfg(test)]
|
||||
Self::read_version_fanout_barrier(&object, index).await;
|
||||
if let Some(delay) = slowtail_fault.as_ref().and_then(|fault| fault.delay_for_disk(index)) {
|
||||
Self::record_metadata_slowtail_fault(&object, index);
|
||||
#[cfg(test)]
|
||||
rename_fanout_barrier::checkpoint(
|
||||
&object,
|
||||
index,
|
||||
rename_fanout_barrier::PHASE_METADATA_SLOWTAIL_FAULT,
|
||||
)
|
||||
.await;
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
read_version_via_coalescer(disk, &org_bucket, &bucket, &object, &version_id, &task_opts, allow_coalescing)
|
||||
.await
|
||||
} else {
|
||||
Err(DiskError::DiskNotFound)
|
||||
};
|
||||
(index, result, response_start.elapsed())
|
||||
};
|
||||
(index, result, response_start.elapsed())
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
diagnostic_context,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::EarlyStop,
|
||||
read,
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
read.await
|
||||
});
|
||||
task_ids[index] = Some(task.id());
|
||||
};
|
||||
|
||||
@@ -35,6 +35,58 @@ struct FileInfoIdentityGroup {
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// Read one key's complete version stack without enumerating its parent.
|
||||
pub(crate) async fn read_object_versions_for_lifecycle(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> disk::error::Result<Option<rustfs_filemeta::MetaCacheEntry>> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() != self.set_drive_count || disks.is_empty() {
|
||||
return Err(DiskError::ErasureReadQuorum);
|
||||
}
|
||||
let disk_object = rustfs_utils::path::encode_dir_object(object);
|
||||
let (mut raw, mut errors) = Self::read_all_raw_file_info(&disks, bucket, &disk_object, false).await;
|
||||
for (metadata, error) in raw.iter_mut().zip(&mut errors) {
|
||||
if metadata.as_ref().is_some_and(|metadata| metadata.buf.is_empty()) {
|
||||
*metadata = None;
|
||||
*error = Some(DiskError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
// An unavailable disk is not evidence that a version group is absent.
|
||||
// Unlike a walk, an exact read must also decide the all-missing case.
|
||||
if raw.iter().all(Option::is_none) {
|
||||
return if errors.iter().all(|error| {
|
||||
matches!(
|
||||
error,
|
||||
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound)
|
||||
)
|
||||
}) {
|
||||
Ok(None)
|
||||
} else {
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
};
|
||||
}
|
||||
let entries = rustfs_filemeta::MetaCacheEntries(
|
||||
raw.into_iter()
|
||||
.map(|raw| {
|
||||
raw.map(|raw| rustfs_filemeta::MetaCacheEntry {
|
||||
name: object.to_owned(),
|
||||
metadata: raw.buf,
|
||||
cached: None,
|
||||
reusable: false,
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
// Reuse the listing's per-version authority, including the object's
|
||||
// actual erasure layout. Never turn unreadable/stale metadata into a
|
||||
// singleton made from the preceding write's ObjectInfo.
|
||||
Self::resolve_listed_versions(bucket, entries, &errors, self.set_drive_count, self.default_parity_count)?
|
||||
.map(Some)
|
||||
.ok_or(DiskError::ErasureReadQuorum)
|
||||
}
|
||||
|
||||
/// Resolve each listed version using the same metadata authority as an exact read.
|
||||
pub(crate) fn resolve_listed_versions(
|
||||
bucket: &str,
|
||||
|
||||
@@ -4778,6 +4778,10 @@ impl DiskHealthEntry {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("committed system metadata write requires repair by its owning writer")]
|
||||
pub(crate) struct SystemMetadataRepairRequired;
|
||||
|
||||
impl SetDisks {
|
||||
pub(in crate::set_disk) async fn persist_delete_marker_purge(
|
||||
&self,
|
||||
@@ -4888,7 +4892,7 @@ impl SetDisks {
|
||||
state = "mrf_durable_admission_failed",
|
||||
bucket, object, version_id, pool_index = self.pool_index, set_index = self.set_index,
|
||||
error = %error,
|
||||
"Partial write repair falling back to the heal channel"
|
||||
"Partial write repair responsibility could not be persisted"
|
||||
);
|
||||
false
|
||||
}
|
||||
@@ -4898,7 +4902,17 @@ impl SetDisks {
|
||||
pub(in crate::set_disk) async fn submit_rename_tail_heal(
|
||||
&self,
|
||||
mut request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
|
||||
) {
|
||||
completion: crate::object_api::WriteCompletion,
|
||||
) -> Result<()> {
|
||||
// System metadata repair remains with the writer's deployment and
|
||||
// transaction fence; a user-bucket MRF authority cannot authorize it.
|
||||
if crate::bucket::utils::is_meta_bucketname(&request.bucket) {
|
||||
return if completion == crate::object_api::WriteCompletion::TailDrainedWithOwnerRepair {
|
||||
Err(Error::other(SystemMetadataRepairRequired))
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
}
|
||||
if let Some(object) = request.object_prefix.as_deref()
|
||||
&& self
|
||||
.persist_partial_write(
|
||||
@@ -4909,10 +4923,10 @@ impl SetDisks {
|
||||
)
|
||||
.await
|
||||
{
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
if request.expected_bucket_incarnation_id.is_none() {
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
request.source = rustfs_heal_contracts::heal_channel::HealRequestSource::Mrf;
|
||||
#[cfg(test)]
|
||||
@@ -4924,11 +4938,12 @@ impl SetDisks {
|
||||
.clone();
|
||||
if let Some(capture) = capture {
|
||||
let _ = capture.send(request);
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let _ = rustfs_heal_contracts::heal_channel::send_heal_request(request).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -10409,6 +10424,13 @@ mod tests {
|
||||
.expect("orphan part should be written");
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
assert_eq!(
|
||||
set.dry_run_reclaim_orphan_data_dirs("bucket", "obj")
|
||||
.await
|
||||
.expect("dry-run orphan cleanup"),
|
||||
1
|
||||
);
|
||||
assert!(object_dir.join(orphan.to_string()).exists(), "dry run must retain the orphan payload");
|
||||
let removed = set
|
||||
.reclaim_orphan_data_dirs("bucket", "obj")
|
||||
.await
|
||||
@@ -10420,6 +10442,67 @@ mod tests {
|
||||
assert!(object_dir.join(STORAGE_FORMAT_FILE).exists(), "metadata must be preserved");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_orphan_data_dirs_drains_empty_discovery_pages() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let live = Uuid::new_v4();
|
||||
let orphan = Uuid::new_v4();
|
||||
let object_dir = dir.path().join("bucket/obj");
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
||||
let live_dir = object_dir.join(live.to_string());
|
||||
let orphan_dir = object_dir.join(orphan.to_string());
|
||||
fs::create_dir_all(&live_dir).await.expect("create live multipart directory");
|
||||
fs::create_dir_all(&orphan_dir).await.expect("create orphan directory");
|
||||
fs::write(orphan_dir.join("part.1"), b"stale")
|
||||
.await
|
||||
.expect("write orphan payload");
|
||||
let parts_dir = live_dir.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
for part in 1..=4200 {
|
||||
std::fs::write(parts_dir.join(format!("part.{part}")), b"live").expect("write live multipart entry");
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("create multipart fixture");
|
||||
|
||||
let first = disk
|
||||
.discover_delete_transactions("bucket", Some("obj"), None, 1)
|
||||
.await
|
||||
.expect("read first discovery page");
|
||||
assert!(first.transactions.is_empty());
|
||||
assert!(first.next.is_some(), "ordinary multipart entries must exhaust the first scan budget");
|
||||
let last = disk
|
||||
.discover_delete_transactions("bucket", Some("obj"), first.next, 1)
|
||||
.await
|
||||
.expect("finish fixture discovery");
|
||||
assert!(
|
||||
last.transactions.is_empty() && last.next.is_none(),
|
||||
"the complete inventory has no transaction evidence"
|
||||
);
|
||||
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
assert_eq!(
|
||||
set.dry_run_reclaim_orphan_data_dirs("bucket", "obj")
|
||||
.await
|
||||
.expect("dry-run paginated orphan cleanup"),
|
||||
1
|
||||
);
|
||||
assert!(orphan_dir.exists(), "dry-run must retain the orphan");
|
||||
assert_eq!(
|
||||
set.reclaim_orphan_data_dirs("bucket", "obj")
|
||||
.await
|
||||
.expect("reclaim after complete discovery"),
|
||||
1
|
||||
);
|
||||
assert!(!orphan_dir.exists(), "both the set preflight and final disk guard must drain empty pages");
|
||||
assert_eq!(
|
||||
fs::read(live_dir.join("part.4200"))
|
||||
.await
|
||||
.expect("read referenced multipart tail"),
|
||||
b"live"
|
||||
);
|
||||
}
|
||||
|
||||
async fn recv_abandoned_parts_trace(
|
||||
trace: &mut rustfs_common::trace_bus::TraceSubscription,
|
||||
bucket: &str,
|
||||
@@ -10633,7 +10716,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_orphan_data_dirs_recovers_committed_delete_marker_without_meta() {
|
||||
async fn reclaim_orphan_data_dirs_defers_delete_marker_without_meta() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let stale = Uuid::new_v4();
|
||||
let transaction = Uuid::new_v4();
|
||||
@@ -10658,8 +10741,105 @@ mod tests {
|
||||
.await
|
||||
.expect("upgrade reclaim should succeed");
|
||||
|
||||
assert_eq!(removed, 1, "the committed delete residue should be reclaimed");
|
||||
assert!(!stale_dir.exists(), "the committed stale data dir should be removed");
|
||||
assert_eq!(removed, 0, "a local marker is not a set-level commit decision");
|
||||
assert_eq!(fs::read(stale_dir.join("part.1")).await.expect("read retained payload"), b"stale");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_orphan_data_dirs_preserves_uuid_prefix_children() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let live = Uuid::new_v4();
|
||||
let prefix = Uuid::new_v4();
|
||||
let child_data = Uuid::new_v4();
|
||||
let object_dir = dir.path().join("bucket/obj");
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
||||
let child_dir = object_dir.join(prefix.to_string()).join("child");
|
||||
write_object_meta_with_data_dirs(&child_dir, "bucket", &format!("obj/{prefix}/child"), &[child_data]).await;
|
||||
let data_dir = child_dir.join(child_data.to_string());
|
||||
fs::create_dir_all(&data_dir).await.expect("create child payload directory");
|
||||
fs::write(data_dir.join("part.1"), b"live child payload")
|
||||
.await
|
||||
.expect("write child payload");
|
||||
let parent_meta = fs::read(object_dir.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("read parent metadata");
|
||||
let child_meta = fs::read(child_dir.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("read child metadata");
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
assert_eq!(set.dry_run_reclaim_orphan_data_dirs("bucket", "obj").await.expect("dry run"), 0);
|
||||
assert_eq!(set.reclaim_orphan_data_dirs("bucket", "obj").await.expect("reclaim"), 0);
|
||||
assert_eq!(
|
||||
fs::read(object_dir.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("read parent metadata"),
|
||||
parent_meta
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(child_dir.join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("read child metadata"),
|
||||
child_meta
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(data_dir.join("part.1")).await.expect("read child payload"),
|
||||
b"live child payload"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_orphan_data_dirs_waits_for_complete_participant_inventory() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let live = Uuid::new_v4();
|
||||
let orphan = Uuid::new_v4();
|
||||
let object_dir = dir.path().join("bucket/obj");
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
||||
let payload_dir = object_dir.join(orphan.to_string());
|
||||
fs::create_dir_all(&payload_dir).await.expect("create orphan directory");
|
||||
fs::write(payload_dir.join("part.1"), b"potentially referenced payload")
|
||||
.await
|
||||
.expect("write payload");
|
||||
let set = make_set_disks_with(vec![Some(disk), None]).await;
|
||||
assert_eq!(set.dry_run_reclaim_orphan_data_dirs("bucket", "obj").await.expect("dry run"), 0);
|
||||
assert_eq!(set.reclaim_orphan_data_dirs("bucket", "obj").await.expect("reclaim"), 0);
|
||||
assert_eq!(
|
||||
fs::read(payload_dir.join("part.1")).await.expect("read retained payload"),
|
||||
b"potentially referenced payload"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reclaim_orphan_data_dirs_defers_payload_referenced_by_neighboring_journal() {
|
||||
let (dir, disk) = make_single_local_disk().await;
|
||||
let live = Uuid::new_v4();
|
||||
let old = Uuid::new_v4();
|
||||
let transaction = Uuid::new_v4();
|
||||
let object_dir = dir.path().join("bucket/obj");
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
||||
let payload_dir = object_dir.join(old.to_string());
|
||||
fs::create_dir_all(&payload_dir).await.expect("create old payload directory");
|
||||
fs::write(payload_dir.join("part.1"), b"rollback payload")
|
||||
.await
|
||||
.expect("write old payload");
|
||||
let transaction_dir = object_dir.join(transaction.to_string());
|
||||
write_object_meta_with_data_dirs(&transaction_dir, "bucket", "obj", &[old]).await;
|
||||
fs::rename(
|
||||
transaction_dir.join(STORAGE_FORMAT_FILE),
|
||||
transaction_dir.join(STORAGE_FORMAT_FILE_BACKUP),
|
||||
)
|
||||
.await
|
||||
.expect("retain rollback metadata");
|
||||
let set = make_set_disks_with(vec![Some(disk)]).await;
|
||||
assert_eq!(set.dry_run_reclaim_orphan_data_dirs("bucket", "obj").await.expect("dry run"), 0);
|
||||
assert_eq!(set.reclaim_orphan_data_dirs("bucket", "obj").await.expect("reclaim"), 0);
|
||||
assert_eq!(
|
||||
fs::read(payload_dir.join("part.1")).await.expect("read rollback payload"),
|
||||
b"rollback payload"
|
||||
);
|
||||
assert!(
|
||||
transaction_dir.join(STORAGE_FORMAT_FILE_BACKUP).exists(),
|
||||
"canonical recovery owns the journal"
|
||||
);
|
||||
}
|
||||
|
||||
// Cross-replica union: a data dir referenced by ANOTHER disk's xl.meta must be
|
||||
|
||||
@@ -36,6 +36,7 @@ pub(crate) struct DeleteRecoverySummary {
|
||||
pub rolled_back: usize,
|
||||
pub blocked: usize,
|
||||
pub pending: usize,
|
||||
pub failed: usize,
|
||||
pub reclaimed: usize,
|
||||
/// Observed removals for this exact key, not aggregated across prefixes.
|
||||
pub removed_versions: HashSet<Uuid>,
|
||||
@@ -52,6 +53,7 @@ pub(crate) struct DeleteRecoveryContext<'a> {
|
||||
/// Only the locked heal path may repair a live marker-creation source.
|
||||
pub repair_marker_source: bool,
|
||||
pub namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub recovery_ticket: Option<crate::store::delete_recovery_runtime::DeleteRecoveryTicket>,
|
||||
pub can_commit: &'a (dyn Fn() -> bool + Sync),
|
||||
}
|
||||
|
||||
@@ -63,6 +65,7 @@ impl<'a> DeleteRecoveryContext<'a> {
|
||||
explicit_delete: false,
|
||||
repair_marker_source: false,
|
||||
namespace_owner: None,
|
||||
recovery_ticket: None,
|
||||
can_commit,
|
||||
}
|
||||
}
|
||||
@@ -75,6 +78,7 @@ impl DeleteRecoverySummary {
|
||||
self.rolled_back += other.rolled_back;
|
||||
self.blocked += other.blocked;
|
||||
self.pending += other.pending;
|
||||
self.failed += other.failed;
|
||||
self.reclaimed += other.reclaimed;
|
||||
}
|
||||
}
|
||||
@@ -307,6 +311,9 @@ impl SetDisks {
|
||||
{
|
||||
// An ordinary missing-key no-op requires no recovery decision.
|
||||
// Do not manufacture witness transactions while a drive is offline.
|
||||
if let Some(ticket) = context.recovery_ticket.as_ref() {
|
||||
self.ctx.delete_recovery_hints().ack(ticket);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let published = join_all(disks.iter().zip(&states).map(|(disk, state)| async {
|
||||
@@ -336,7 +343,15 @@ impl SetDisks {
|
||||
}
|
||||
// Physical cleanup is retryable after the certified decision. An
|
||||
// unavailable participant retains the decision for its eventual return.
|
||||
let _ = self.recover_delete_key(bucket, object, context).await;
|
||||
let ticket = context.recovery_ticket.clone();
|
||||
if let Ok(summary) = self.recover_delete_key(bucket, object, context).await
|
||||
&& summary.pending == 0
|
||||
&& summary.blocked == 0
|
||||
&& summary.failed == 0
|
||||
&& let Some(ticket) = ticket.as_ref()
|
||||
{
|
||||
self.ctx.delete_recovery_hints().ack(ticket);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -363,6 +378,7 @@ impl SetDisks {
|
||||
}
|
||||
let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, None))).await;
|
||||
let mut keys = BTreeSet::new();
|
||||
let mut summary = DeleteRecoverySummary::default();
|
||||
for page in pages {
|
||||
match page {
|
||||
Ok(ids) => {
|
||||
@@ -373,18 +389,24 @@ impl SetDisks {
|
||||
);
|
||||
}
|
||||
Err(error) if unavailable(&error) => {
|
||||
return Ok(DeleteRecoverySummary {
|
||||
pending: 1,
|
||||
..Default::default()
|
||||
});
|
||||
summary.pending += 1;
|
||||
}
|
||||
Err(DiskError::FileCorrupt) => summary.failed += 1,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
let mut summary = DeleteRecoverySummary::default();
|
||||
// Every key requires complete participation. Avoid taking each key's
|
||||
// lock and repeating discovery while this sweep already lacks a peer.
|
||||
if summary.pending > 0 {
|
||||
return Ok(summary);
|
||||
}
|
||||
// Lock order: caller's bucket lifecycle fence, then one exact key.
|
||||
for object in keys {
|
||||
summary.add(self.recover_delete_object(bucket, &object, context.clone()).await?);
|
||||
match self.recover_delete_object(bucket, &object, context.clone()).await {
|
||||
Ok(result) => summary.add(result),
|
||||
Err(DiskError::FileCorrupt) => summary.failed += 1,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
@@ -428,6 +450,7 @@ impl SetDisks {
|
||||
repair_marker_source,
|
||||
namespace_owner,
|
||||
can_commit,
|
||||
recovery_ticket: _,
|
||||
} = context;
|
||||
let disks = self.disk_inventory().await;
|
||||
let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
|
||||
@@ -1054,6 +1077,131 @@ mod tests {
|
||||
assert_eq!(body(&store, bucket, None).await, original);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_corrupt_bucket_does_not_prevent_other_bucket_rollback() {
|
||||
let bad_bucket = "recover-corrupt-bucket";
|
||||
let good_bucket = "recover-good-bucket";
|
||||
let (dirs, store, disks) = fixture(bad_bucket).await;
|
||||
store
|
||||
.make_bucket(good_bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create healthy bucket");
|
||||
put(&store, bad_bucket, &[61; 4096], false).await;
|
||||
let bad_transaction = applied(&store, &disks, bad_bucket, None, disks.len()).await;
|
||||
let good_bytes = vec![62; 4096];
|
||||
put(&store, good_bucket, &good_bytes, false).await;
|
||||
applied(&store, &disks, good_bucket, None, disks.len()).await;
|
||||
for dir in &dirs {
|
||||
std::fs::write(
|
||||
dir.path()
|
||||
.join(bad_bucket)
|
||||
.join("object")
|
||||
.join(bad_transaction.to_string())
|
||||
.join("delete-prepared"),
|
||||
b"{broken",
|
||||
)
|
||||
.expect("inject malformed preparation journal");
|
||||
}
|
||||
|
||||
let failed = store
|
||||
.recover_delete_transactions(bad_bucket, &HealOpts::default())
|
||||
.await
|
||||
.expect("bucket corruption is an explicit partial recovery result");
|
||||
assert_eq!(failed.delete_recovery.expect("recovery status").failed, 1);
|
||||
let error = store
|
||||
.recover_delete_transactions_at_incarnation(bad_bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect_err("exact object recovery must retain typed corruption");
|
||||
assert!(matches!(error, crate::error::StorageError::FileCorrupt));
|
||||
let recovered = store
|
||||
.recover_delete_transactions(good_bucket, &HealOpts::default())
|
||||
.await
|
||||
.expect("another bucket must remain recoverable");
|
||||
assert!(recovered.detail.contains("rolled_back=1"));
|
||||
assert!(!recovered.delete_recovery.expect("healthy recovery status").has_unresolved());
|
||||
assert_eq!(body(&store, good_bucket, None).await, good_bytes);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_corrupt_key_does_not_prevent_other_key_rollback() {
|
||||
let bucket = "recover-corrupt-key";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
let healthy_bytes = vec![63; 4096];
|
||||
put(&store, bucket, &healthy_bytes, false).await;
|
||||
applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![64; 4096]);
|
||||
store
|
||||
.put_object(
|
||||
bucket,
|
||||
"a-corrupt",
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write corrupt-key fixture");
|
||||
let transaction = Uuid::new_v4();
|
||||
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("bucket identity");
|
||||
for disk in &disks {
|
||||
disk.delete_version_recoverable(
|
||||
bucket,
|
||||
"a-corrupt",
|
||||
FileInfo {
|
||||
name: "a-corrupt".to_owned(),
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
delete_recovery_bucket_incarnation: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("prepare corrupt-key fixture");
|
||||
}
|
||||
let damaged = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("a-corrupt")
|
||||
.join(transaction.to_string())
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE_BACKUP);
|
||||
std::fs::write(&damaged, b"corrupt metadata").expect("damage one backup while retaining discoverable journal");
|
||||
|
||||
let report = store
|
||||
.recover_delete_transactions(bucket, &HealOpts::default())
|
||||
.await
|
||||
.expect("per-key corruption must not abort the remaining prefix");
|
||||
|
||||
assert_eq!(report.delete_recovery.expect("recovery status").failed, 1);
|
||||
assert!(report.detail.contains("rolled_back=1"));
|
||||
assert_eq!(std::fs::read(damaged).expect("corrupt proof is retained"), b"corrupt metadata");
|
||||
assert_eq!(body(&store, bucket, None).await, healthy_bytes);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_offline_bucket_reports_pending_without_hiding_exact_object_error() {
|
||||
let bucket = "recover-offline-status";
|
||||
let (_dirs, store, _disks) = fixture(bucket).await;
|
||||
store.pools[0].disk_set[0].disks.write().await[3] = None;
|
||||
|
||||
let result = store
|
||||
.recover_delete_transactions(bucket, &HealOpts::default())
|
||||
.await
|
||||
.expect("offline recovery must expose a pending summary even with no discovered transactions");
|
||||
let status = result.delete_recovery.expect("offline status is retained");
|
||||
assert_eq!(status.pending, 1);
|
||||
assert_eq!(status.failed, 0);
|
||||
assert!(status.has_unresolved());
|
||||
let error = store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect_err("exact-object workers require a typed retry error");
|
||||
assert!(matches!(error, crate::error::StorageError::ErasureReadQuorum));
|
||||
}
|
||||
|
||||
fn snapshot(root: &std::path::Path) -> BTreeMap<std::path::PathBuf, Vec<u8>> {
|
||||
let mut files = BTreeMap::new();
|
||||
let mut pending = vec![root.to_path_buf()];
|
||||
@@ -2033,13 +2181,18 @@ mod tests {
|
||||
}
|
||||
}
|
||||
let error = store
|
||||
.recover_delete_transactions(bucket, &HealOpts::default())
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect_err("cyclic history cannot settle");
|
||||
assert!(
|
||||
matches!(error, crate::error::StorageError::FileCorrupt),
|
||||
"typed corruption must reject cyclic evidence: {error:?}"
|
||||
);
|
||||
let report = store
|
||||
.recover_delete_transactions(bucket, &HealOpts::default())
|
||||
.await
|
||||
.expect("a bucket report must retain its corrupt-key status");
|
||||
assert_eq!(report.delete_recovery.expect("corrupt-key status").failed, 1);
|
||||
for (path, bytes) in evidence {
|
||||
assert_eq!(std::fs::read(path).expect("all proof retained"), bytes);
|
||||
}
|
||||
|
||||
@@ -800,8 +800,15 @@ impl SetDisks {
|
||||
false,
|
||||
true,
|
||||
self.default_parity_count,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let metadata_read = crate::cluster::rpc::runtime_sources::with_read_version_request_caller_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::Heal,
|
||||
metadata_read,
|
||||
)
|
||||
.await?;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
let metadata_read = metadata_read.await?;
|
||||
let (parts_metadata, errs, _) = metadata_read.into_legacy();
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&parts_metadata, &errs, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
|
||||
@@ -63,7 +63,6 @@ use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::disk::DiskAPI;
|
||||
#[cfg(test)]
|
||||
use crate::disk::DiskOption;
|
||||
#[cfg(test)]
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
#[cfg(test)]
|
||||
use crate::disk::new_disk;
|
||||
@@ -231,6 +230,60 @@ pub(crate) struct StaleMultipartCleanupGuard {
|
||||
lock_guard: ObjectLockDiagGuard,
|
||||
}
|
||||
|
||||
/// Exact-key recovery is admitted only after every disk proves that the upload
|
||||
/// metadata is absent. Keep the checked disk generation and upload lock through
|
||||
/// deletion; a quorum miss is not proof that an upload is an orphan.
|
||||
/// Final object data, completion replay identity, and old-data cleanup receipts
|
||||
/// live under the destination bucket/object, never this staging path. This also
|
||||
/// permits removing an exact orphan from a previous bucket incarnation without
|
||||
/// claiming that its UploadID belongs to the current incarnation.
|
||||
pub(crate) struct OrphanMultipartCleanupGuard {
|
||||
set: Arc<SetDisks>,
|
||||
disks: Vec<Option<DiskStore>>,
|
||||
upload_path: String,
|
||||
lock_guard: Arc<ObjectLockDiagGuard>,
|
||||
exists: bool,
|
||||
}
|
||||
|
||||
impl OrphanMultipartCleanupGuard {
|
||||
pub(crate) async fn delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<bool> {
|
||||
fence_commit_on_lock_loss(Some(&self.lock_guard), "orphan_multipart_cleanup", &self.upload_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
if !self.exists {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let results = join_all(self.disks.iter().map(|disk| async {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
match disk
|
||||
.delete(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&self.upload_path,
|
||||
crate::disk::DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::FileNotFound | DiskError::PathNotFound | DiskError::VolumeNotFound) => Ok(()),
|
||||
result => result,
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
self.set.record_capacity_scope_if_needed(None, &self.disks);
|
||||
let errors = results.into_iter().map(|result| result.err()).collect::<Vec<_>>();
|
||||
if let Some(err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, self.disks.len()) {
|
||||
return Err(to_object_err(err.into(), vec![bucket, object]));
|
||||
}
|
||||
fence_commit_on_lock_loss(Some(&self.lock_guard), "orphan_multipart_cleanup", &self.upload_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct DataMovementMultipartAbortGuard {
|
||||
upload_path: String,
|
||||
write_quorum: Option<usize>,
|
||||
@@ -557,6 +610,34 @@ fn multipart_bucket_incarnation_matches(metadata: &HashMap<String, String>, expe
|
||||
matches!(multipart_bucket_incarnation_id(metadata), Ok(Some(actual)) if actual == expected)
|
||||
}
|
||||
|
||||
fn orphan_multipart_upload_id_is_canonical(upload_id: &str) -> bool {
|
||||
let Ok(decoded) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes()) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(decoded) = std::str::from_utf8(&decoded) else {
|
||||
return false;
|
||||
};
|
||||
let Some((_, raw)) = decoded.split_once('.') else {
|
||||
return false;
|
||||
};
|
||||
let Some((uuid, timestamp)) = raw.split_once('x') else {
|
||||
return false;
|
||||
};
|
||||
let Ok(id) = Uuid::parse_str(uuid) else {
|
||||
return false;
|
||||
};
|
||||
if id.is_nil() || id.to_string() != uuid {
|
||||
return false;
|
||||
}
|
||||
let Ok(timestamp_ns) = timestamp.parse::<i128>() else {
|
||||
return false;
|
||||
};
|
||||
if timestamp_ns.to_string() != timestamp {
|
||||
return false;
|
||||
}
|
||||
OffsetDateTime::from_unix_timestamp_nanos(timestamp_ns).is_ok()
|
||||
}
|
||||
|
||||
fn validate_multipart_bucket_incarnation(
|
||||
metadata: &HashMap<String, String>,
|
||||
bucket: &str,
|
||||
@@ -1246,6 +1327,114 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn lock_orphan_multipart_cleanup(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
opts: &ObjectOptions,
|
||||
held_guards: &[OrphanMultipartCleanupGuard],
|
||||
) -> Result<OrphanMultipartCleanupGuard> {
|
||||
let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
|
||||
if opts.data_movement
|
||||
|| opts.no_lock
|
||||
|| self.local_lock_manager.is_disabled()
|
||||
|| !orphan_multipart_upload_id_is_canonical(upload_id)
|
||||
{
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
// This path is derived from the caller's exact bucket/key and a canonical
|
||||
// UUID/timestamp upload identity, never from the shared MPU directory list.
|
||||
let upload_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
// Pool labels do not partition the raw ObjectKey sent to lock clients.
|
||||
// Reuse the actual owner only for this exact path and proven backend;
|
||||
// independent distributed domains still need their own guards.
|
||||
let distributed = self.ctx.is_dist_erasure().await;
|
||||
let mut lock_guard = None;
|
||||
for held in held_guards {
|
||||
if held.upload_path != upload_path {
|
||||
continue;
|
||||
}
|
||||
let shared_domain = match (distributed, held.set.ctx.is_dist_erasure().await) {
|
||||
(false, false) => self.shares_namespace_lock_domain(&held.set).await,
|
||||
(true, true) => {
|
||||
let same = super::super::same_distributed_lock_domain(&self.lockers, &held.set.lockers);
|
||||
if !same
|
||||
&& self
|
||||
.lockers
|
||||
.iter()
|
||||
.any(|client| held.set.lockers.iter().any(|held_client| Arc::ptr_eq(client, held_client)))
|
||||
{
|
||||
// A guard for only part of another quorum is not proof
|
||||
// of ownership there, and acquiring it again may wait
|
||||
// on ourselves. Preserve all staging in this topology.
|
||||
return Err(Error::other("orphan multipart cleanup cannot cover partially overlapping lock domains"));
|
||||
}
|
||||
same
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if shared_domain {
|
||||
lock_guard = Some(Arc::clone(&held.lock_guard));
|
||||
break;
|
||||
}
|
||||
}
|
||||
let lock_guard = match lock_guard {
|
||||
Some(guard) => guard,
|
||||
None => Arc::new(
|
||||
self.acquire_write_lock_diag("orphan_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await?,
|
||||
),
|
||||
};
|
||||
fence_commit_on_lock_loss(Some(&lock_guard), "orphan_multipart_cleanup", &upload_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.len() != self.set_drive_count || disks.is_empty() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
// Raw full fan-out cannot quorum-short-circuit or interpret an empty
|
||||
// version list as absence of the physical metadata file.
|
||||
let (_, errors) = Self::read_all_raw_file_info(&disks, RUSTFS_META_MULTIPART_BUCKET, &upload_path, false).await;
|
||||
if errors.len() != disks.len() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
for error in errors {
|
||||
match error {
|
||||
Some(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
|
||||
// Even one valid metadata copy may still be recoverable. Never
|
||||
// convert a partial, corrupt, or unreachable upload into debris.
|
||||
None => return Err(invalid_upload()),
|
||||
Some(error) => return Err(to_object_err(error.into(), vec![bucket, object, upload_id])),
|
||||
}
|
||||
}
|
||||
let mut exists = false;
|
||||
for disk in &disks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
};
|
||||
match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await {
|
||||
Ok(entries) => {
|
||||
// Raw metadata reads report a directory named xl.meta as
|
||||
// FileNotFound. Its physical presence is unknown/corrupt
|
||||
// metadata, not proof that this staging can be deleted.
|
||||
if entries.iter().any(|entry| entry.trim_end_matches('/') == STORAGE_FORMAT_FILE) {
|
||||
return Err(to_object_err(DiskError::FileCorrupt.into(), vec![bucket, object, upload_id]));
|
||||
}
|
||||
exists = true;
|
||||
}
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
|
||||
Err(error) => return Err(to_object_err(error.into(), vec![bucket, object, upload_id])),
|
||||
}
|
||||
}
|
||||
Ok(OrphanMultipartCleanupGuard {
|
||||
set: self.clone(),
|
||||
disks,
|
||||
upload_path,
|
||||
lock_guard,
|
||||
exists,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||
&self,
|
||||
@@ -1678,6 +1867,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
drop(admission_guard);
|
||||
}
|
||||
|
||||
let mut commit_owns_cleanup = false;
|
||||
let result: Result<PartInfo> = async {
|
||||
let erasure =
|
||||
Arc::new(coding::Erasure::try_new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size)
|
||||
@@ -1881,120 +2071,171 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.await?;
|
||||
(Some(upload_guard), Some(part_guard))
|
||||
};
|
||||
let (commit_fi, _) = self
|
||||
.check_upload_id_exists_with_opts(bucket, object, upload_id, multipart_size_limit.is_some(), opts)
|
||||
.await?;
|
||||
let commit_size_limit = multipart_size_limit_from_metadata(&commit_fi.metadata)?;
|
||||
if commit_size_limit != multipart_size_limit {
|
||||
return Err(Error::InvalidArgument(
|
||||
"multipart upload".to_string(),
|
||||
SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
|
||||
"size limit metadata changed or is missing".to_string(),
|
||||
));
|
||||
}
|
||||
ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
|
||||
ensure_multipart_bucket_incarnation(
|
||||
&self.ctx,
|
||||
&commit_fi,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
|
||||
fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
|
||||
fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_outer_lock",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
if let Some(limit) = commit_size_limit {
|
||||
let current_size = self
|
||||
.current_multipart_logical_size(bucket, object, upload_id, &upload_id_path, &commit_fi, part_id)
|
||||
.await?;
|
||||
let candidate_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
admitted_multipart_size(current_size, candidate_size, limit)?;
|
||||
}
|
||||
|
||||
let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(
|
||||
&[self.pool_index],
|
||||
DecommissionCapacityAdmission::ExistingMultipart,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await;
|
||||
if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_decommission_capacity",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
let rename_result = self
|
||||
.rename_part(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
part_info_buff.into(),
|
||||
write_quorum,
|
||||
Some(MultipartWriteQuorumContext {
|
||||
stage: MULTIPART_WRITE_QUORUM_RENAME_PART,
|
||||
// Only guards owned by this operation may outlive its caller. Internal
|
||||
// data-movement and borrowed-lock callers retain their synchronous
|
||||
// lifetime; a fence by itself cannot transfer the outer lock lease.
|
||||
let detach_commit_owner = _upload_commit_guard.is_some()
|
||||
&& opts.namespace_lock_fence.is_none()
|
||||
&& (opts.bucket_lifecycle_lock_fence.is_none() || opts.bucket_lifecycle_lock_owner.is_some());
|
||||
let commit_set = self.clone();
|
||||
let commit_bucket = bucket.to_owned();
|
||||
let commit_object = object.to_owned();
|
||||
let commit_upload_id = upload_id.to_owned();
|
||||
let commit_upload_id_path = upload_id_path.clone();
|
||||
let commit_tmp_part = tmp_part.clone();
|
||||
let commit_opts = opts.clone();
|
||||
commit_owns_cleanup = true;
|
||||
// The body is already staged. Keep every commit guard through the
|
||||
// prepare, publish, settlement/rollback and temporary cleanup I/O,
|
||||
// even when the HTTP request stops awaiting this task.
|
||||
let commit = async move {
|
||||
let _upload_commit_guard = _upload_commit_guard;
|
||||
let _part_commit_guard = _part_commit_guard;
|
||||
let _capped_staging_guard = _capped_staging_guard;
|
||||
let bucket = commit_bucket.as_str();
|
||||
let object = commit_object.as_str();
|
||||
let upload_id = commit_upload_id.as_str();
|
||||
let upload_id_path = commit_upload_id_path;
|
||||
let opts = &commit_opts;
|
||||
let result: Result<PartInfo> = async {
|
||||
let (commit_fi, _) = commit_set
|
||||
.check_upload_id_exists_with_opts(bucket, object, upload_id, multipart_size_limit.is_some(), opts)
|
||||
.await?;
|
||||
let commit_size_limit = multipart_size_limit_from_metadata(&commit_fi.metadata)?;
|
||||
if commit_size_limit != multipart_size_limit {
|
||||
return Err(Error::InvalidArgument(
|
||||
"multipart upload".to_string(),
|
||||
SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
|
||||
"size limit metadata changed or is missing".to_string(),
|
||||
));
|
||||
}
|
||||
ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
|
||||
ensure_multipart_bucket_incarnation(
|
||||
&commit_set.ctx,
|
||||
&commit_fi,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
part_number: Some(part_id),
|
||||
}),
|
||||
)
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
|
||||
fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
|
||||
fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_outer_lock",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
if let Some(limit) = commit_size_limit {
|
||||
let current_size = commit_set
|
||||
.current_multipart_logical_size(bucket, object, upload_id, &upload_id_path, &commit_fi, part_id)
|
||||
.await?;
|
||||
let candidate_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
admitted_multipart_size(current_size, candidate_size, limit)?;
|
||||
}
|
||||
|
||||
let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(
|
||||
&[commit_set.pool_index],
|
||||
DecommissionCapacityAdmission::ExistingMultipart,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await;
|
||||
if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_decommission_capacity",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
let rename_result = commit_set
|
||||
.rename_part(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
part_info_buff.into(),
|
||||
write_quorum,
|
||||
Some(MultipartWriteQuorumContext {
|
||||
stage: MULTIPART_WRITE_QUORUM_RENAME_PART,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
part_number: Some(part_id),
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
drop(decommission_capacity_guard);
|
||||
let _ = rename_result?;
|
||||
#[cfg(test)]
|
||||
observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
|
||||
|
||||
let ret: PartInfo = PartInfo {
|
||||
etag: Some(etag.clone()),
|
||||
part_num: part_id,
|
||||
last_mod: Some(OffsetDateTime::now_utc()),
|
||||
size: w_size,
|
||||
actual_size,
|
||||
};
|
||||
|
||||
// error!("put_object_part ret {:?}", &ret);
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
.await;
|
||||
drop(decommission_capacity_guard);
|
||||
let _ = rename_result?;
|
||||
#[cfg(test)]
|
||||
observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
|
||||
drop(_part_commit_guard);
|
||||
drop(_upload_commit_guard);
|
||||
|
||||
let ret: PartInfo = PartInfo {
|
||||
etag: Some(etag.clone()),
|
||||
part_num: part_id,
|
||||
last_mod: Some(OffsetDateTime::now_utc()),
|
||||
size: w_size,
|
||||
actual_size,
|
||||
if result.is_err()
|
||||
&& let Err(err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_part).await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_MULTIPART,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
result = "cleanup_failed",
|
||||
bucket,
|
||||
object,
|
||||
error = ?err,
|
||||
"failed to clean multipart commit staging"
|
||||
);
|
||||
}
|
||||
result
|
||||
};
|
||||
|
||||
// error!("put_object_part ret {:?}", &ret);
|
||||
|
||||
Ok(ret)
|
||||
if detach_commit_owner {
|
||||
tokio::spawn(commit)
|
||||
.await
|
||||
.map_err(|err| Error::other_with_context("put_object_part commit task failed", err))?
|
||||
} else {
|
||||
commit.await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
|
||||
if result.is_err()
|
||||
if !commit_owns_cleanup
|
||||
&& result.is_err()
|
||||
&& let Err(err) = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_part).await
|
||||
{
|
||||
warn!(tmp_part = %tmp_part, error = ?err, "failed to cleanup multipart temporary data");
|
||||
@@ -3385,7 +3626,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_multipart_commit(&commit_bucket, &commit_object, MultipartCommitPause::BeforeQuotaRename).await;
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| write_context.is_lock_lost()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
@@ -3440,7 +3680,6 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
verify_object_transaction_epoch_fence(&commit_set, &commit_bucket, &commit_object, expected).await?;
|
||||
}
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| write_context.is_lock_lost()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
@@ -4233,6 +4472,300 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_multipart_identity_requires_canonical_upload_id() {
|
||||
let created = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10);
|
||||
let id = Uuid::new_v4();
|
||||
let timestamp = (created + time::Duration::seconds(1)).unix_timestamp_nanos();
|
||||
let encode = |raw: &str| runtime_sources::deployment_upload_id(raw);
|
||||
let upload_id = encode(&format!("{id}x{timestamp}"));
|
||||
assert!(orphan_multipart_upload_id_is_canonical(&upload_id));
|
||||
for raw in [
|
||||
format!("{}x{timestamp}", Uuid::nil()),
|
||||
format!("{}x{timestamp}", id.simple()),
|
||||
format!("{id}x0{timestamp}"),
|
||||
format!("{id}x+{timestamp}"),
|
||||
format!("{id}x{timestamp}/../another-upload"),
|
||||
format!("../{id}x{timestamp}"),
|
||||
format!("{id}x{}", i128::MAX),
|
||||
] {
|
||||
assert!(
|
||||
!orphan_multipart_upload_id_is_canonical(&encode(&raw)),
|
||||
"noncanonical upload identity must be rejected: {raw}"
|
||||
);
|
||||
}
|
||||
assert!(!orphan_multipart_upload_id_is_canonical("not-base64"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_requires_every_disk_to_prove_metadata_absence() {
|
||||
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "orphan-multipart-proof";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
|
||||
let metadata_path = format!("{upload_path}/{STORAGE_FORMAT_FILE}");
|
||||
let leftover_path = format!("{upload_path}/part.late");
|
||||
for disk in &disks {
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path, Bytes::from_static(b"late part"))
|
||||
.await
|
||||
.expect("write late part fixture");
|
||||
}
|
||||
for disk in &disks[..3] {
|
||||
disk.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
|
||||
.await
|
||||
.expect("remove majority metadata");
|
||||
}
|
||||
let opts = ObjectOptions::default();
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"one surviving metadata copy must block orphan recovery"
|
||||
);
|
||||
disks[3]
|
||||
.write_all(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Bytes::from_static(b"corrupt metadata"))
|
||||
.await
|
||||
.expect("corrupt final metadata copy");
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"corrupt metadata must not be treated as absence"
|
||||
);
|
||||
disks[3]
|
||||
.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
|
||||
.await
|
||||
.expect("remove final metadata copy");
|
||||
set.disks.write().await[3] = None;
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"an unreachable disk must not be hidden by absence quorum"
|
||||
);
|
||||
set.disks.write().await[3] = Some(disks[3].clone());
|
||||
for disk in &disks {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path)
|
||||
.await
|
||||
.expect("failed preflight must preserve every part"),
|
||||
Bytes::from_static(b"late part")
|
||||
);
|
||||
}
|
||||
|
||||
let guard = set
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.expect("all disks now prove metadata absence");
|
||||
assert!(guard.delete(bucket, object, &opts).await.expect("recover exact staging"));
|
||||
drop(guard);
|
||||
for disk in &disks {
|
||||
assert!(matches!(
|
||||
disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound)
|
||||
));
|
||||
}
|
||||
let guard = set
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.expect("already absent staging remains provably absent");
|
||||
assert!(
|
||||
!guard
|
||||
.delete(bucket, object, &opts)
|
||||
.await
|
||||
.expect("absence does not report recovery")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "orphan-multipart-directory";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
|
||||
let metadata_path = format!("{upload_path}/{STORAGE_FORMAT_FILE}");
|
||||
let leftover_path = format!("{upload_path}/part.late");
|
||||
for disk in &disks {
|
||||
disk.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
|
||||
.await
|
||||
.expect("remove upload metadata");
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path, Bytes::from_static(b"late part"))
|
||||
.await
|
||||
.expect("write leftover part");
|
||||
}
|
||||
let metadata_directory = temp_dirs[3].path().join(RUSTFS_META_MULTIPART_BUCKET).join(&metadata_path);
|
||||
tokio::fs::create_dir_all(&metadata_directory)
|
||||
.await
|
||||
.expect("replace one missing metadata file with a directory");
|
||||
let (_, errors) =
|
||||
SetDisks::read_all_raw_file_info(&set.disks.read().await, RUSTFS_META_MULTIPART_BUCKET, &upload_path, false).await;
|
||||
assert!(
|
||||
errors.iter().all(|error| matches!(error, Some(DiskError::FileNotFound))),
|
||||
"raw metadata reads alone must reproduce the directory-as-absence case: {errors:?}"
|
||||
);
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &ObjectOptions::default(), &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"one physical metadata directory must block cleanup on every disk"
|
||||
);
|
||||
assert!(metadata_directory.is_dir(), "failed preflight must preserve the metadata directory");
|
||||
for disk in &disks {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path)
|
||||
.await
|
||||
.expect("failed preflight must preserve every leftover part"),
|
||||
Bytes::from_static(b"late part")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum OrphanMultipartLockDomainCase {
|
||||
Shared,
|
||||
Independent,
|
||||
PartiallyOverlapping,
|
||||
}
|
||||
|
||||
async fn check_orphan_multipart_distributed_lock_owners(case: OrphanMultipartLockDomainCase) {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let first_client: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let second_client: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let second_clients = match case {
|
||||
OrphanMultipartLockDomainCase::Shared => vec![first_client.clone()],
|
||||
OrphanMultipartLockDomainCase::Independent => vec![second_client],
|
||||
OrphanMultipartLockDomainCase::PartiallyOverlapping => vec![first_client.clone(), second_client],
|
||||
};
|
||||
let (first_dirs, first_disks, first_set) =
|
||||
hermetic_set_disks_with_lockers_and_ctx(4, 0, 2, vec![first_client], ctx.clone()).await;
|
||||
let (second_dirs, second_disks, second_set) = hermetic_set_disks_with_lockers_and_ctx(4, 1, 2, second_clients, ctx).await;
|
||||
let bucket = "orphan-multipart-dist-locks";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&first_disks, bucket).await;
|
||||
make_bucket_on_all(&second_disks, bucket).await;
|
||||
let upload_id = runtime_sources::deployment_upload_id(&format!(
|
||||
"{}x{}",
|
||||
Uuid::new_v4(),
|
||||
OffsetDateTime::now_utc().unix_timestamp_nanos()
|
||||
));
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
for directory in first_dirs.iter().chain(second_dirs.iter()) {
|
||||
let staging = directory.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path);
|
||||
tokio::fs::create_dir_all(&staging)
|
||||
.await
|
||||
.expect("create exact orphan staging");
|
||||
tokio::fs::write(staging.join("part.late"), b"late part")
|
||||
.await
|
||||
.expect("write orphan part");
|
||||
}
|
||||
let opts = ObjectOptions::default();
|
||||
let first_guard = first_set
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &upload_id, &opts, &[])
|
||||
.await
|
||||
.expect("first domain has an orphan");
|
||||
let second_result = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
second_set.lock_orphan_multipart_cleanup(bucket, object, &upload_id, &opts, std::slice::from_ref(&first_guard)),
|
||||
)
|
||||
.await
|
||||
.expect("lock-domain admission must not wait on a guard it already owns");
|
||||
if case == OrphanMultipartLockDomainCase::PartiallyOverlapping {
|
||||
let error = match second_result {
|
||||
Ok(_) => panic!("one existing quorum cannot authorize a partially overlapping domain"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.to_string().contains("partially overlapping lock domains"));
|
||||
for directory in first_dirs.iter().chain(second_dirs.iter()) {
|
||||
assert!(
|
||||
directory
|
||||
.path()
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_path)
|
||||
.join("part.late")
|
||||
.exists(),
|
||||
"uncertain lock coverage must preserve every physical part"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let second_guard = second_result.expect("the second pool has a proven lock domain");
|
||||
assert_eq!(
|
||||
Arc::ptr_eq(&first_guard.lock_guard, &second_guard.lock_guard),
|
||||
case == OrphanMultipartLockDomainCase::Shared,
|
||||
"only the same exact backend may share a real upload guard across pool labels"
|
||||
);
|
||||
let second_namespace = second_set
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut second_writer = Box::pin(second_namespace.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(futures::poll!(&mut second_writer).is_pending());
|
||||
if case == OrphanMultipartLockDomainCase::Shared {
|
||||
drop(first_guard);
|
||||
assert!(
|
||||
futures::poll!(&mut second_writer).is_pending(),
|
||||
"the second pool must retain the shared real owner after the first guard drops"
|
||||
);
|
||||
drop(second_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), second_writer)
|
||||
.await
|
||||
.expect("the shared backend should release after the last owner")
|
||||
.expect("shared-domain writer resumes"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let first_namespace = first_set
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut first_writer = Box::pin(first_namespace.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(futures::poll!(&mut first_writer).is_pending());
|
||||
drop(first_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), first_writer)
|
||||
.await
|
||||
.expect("first domain should release independently")
|
||||
.expect("first writer resumes"),
|
||||
);
|
||||
assert!(futures::poll!(&mut second_writer).is_pending(), "the second real owner must remain held");
|
||||
drop(second_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), second_writer)
|
||||
.await
|
||||
.expect("second domain should release with its owner")
|
||||
.expect("second writer resumes"),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_keeps_separate_distributed_lock_owners() {
|
||||
check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::Independent).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_reuses_shared_distributed_owner_across_pools() {
|
||||
check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::Shared).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_rejects_partially_overlapping_distributed_domains() {
|
||||
check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::PartiallyOverlapping).await;
|
||||
}
|
||||
|
||||
async fn stage_upload_with_create_opts(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
@@ -6370,6 +6903,76 @@ mod tests {
|
||||
.expect("abort should delete the upload after UploadPart releases the lock");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn cancelled_put_object_part_keeps_upload_lock_through_commit() {
|
||||
for pause in [
|
||||
MultipartCommitPause::PutPartAfterCapacityAdmission,
|
||||
MultipartCommitPause::PutPartAfterRename,
|
||||
] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-cancelled-part-commit-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, pause);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
let put_upload_id = upload_id.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x45; 4096]);
|
||||
put_store
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
put.abort();
|
||||
assert!(
|
||||
put.await
|
||||
.expect_err("the request waiter should remain cancellable")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let mut abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut abort).await.is_err(),
|
||||
"Abort must not acquire the upload lock while a cancelled request's commit is paused at {pause:?}"
|
||||
);
|
||||
barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(10), abort)
|
||||
.await
|
||||
.expect("Abort should finish after the commit drains")
|
||||
.expect("Abort task should not panic")
|
||||
.expect("Abort should delete the committed part staging");
|
||||
for disk in &disk_stores {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"the upload directory must be absent after the commit and Abort finish"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn put_object_part_different_part_numbers_commit_concurrently() {
|
||||
@@ -9799,6 +10402,18 @@ mod tests {
|
||||
"the post-commit crash must leave the upload listable for reclamation"
|
||||
);
|
||||
|
||||
let cleanup = set_disks
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &u_new, &ObjectOptions::default(), &[])
|
||||
.await
|
||||
.expect("the committed upload has no staging metadata on any disk");
|
||||
assert!(
|
||||
cleanup
|
||||
.delete(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recover completed upload staging")
|
||||
);
|
||||
drop(cleanup);
|
||||
|
||||
// The commit rename already consumed the upload's xl.meta, so the
|
||||
// retry cannot read staging. It must still return the committed
|
||||
// object: AWS CompleteMultipartUpload is idempotent for the same
|
||||
@@ -9817,21 +10432,6 @@ mod tests {
|
||||
!upload_is_listed(&set_disks, bucket, object, &u_new).await,
|
||||
"the idempotent retry must reclaim staging left behind by the crash"
|
||||
);
|
||||
|
||||
// Reclaim the leftover exactly as the production tail does: delete_all
|
||||
// on the upload path (abort_multipart_upload cannot — the upload's
|
||||
// metadata is gone, so it too answers InvalidUploadID).
|
||||
let upload_dir = SetDisks::get_upload_id_dir(bucket, object, &u_new);
|
||||
set_disks
|
||||
.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_dir)
|
||||
.await
|
||||
.expect("sweeping the leftover upload staging should succeed");
|
||||
assert!(
|
||||
!upload_is_listed(&set_disks, bucket, object, &u_new).await,
|
||||
"reclaiming the upload must drop it from the listing"
|
||||
);
|
||||
let (body_after, _) = read_object(&set_disks, bucket, object).await;
|
||||
assert_eq!(body_after, new, "reclaiming the leftover upload must not disturb the committed object");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -9878,6 +10478,18 @@ mod tests {
|
||||
.expect("the failed post-commit completion should release its object guard"),
|
||||
);
|
||||
|
||||
let cleanup = set_disks
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &u_new, &ObjectOptions::default(), &[])
|
||||
.await
|
||||
.expect("committed staging has no upload metadata");
|
||||
assert!(
|
||||
cleanup
|
||||
.delete(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("orphan recovery must only remove upload staging")
|
||||
);
|
||||
drop(cleanup);
|
||||
|
||||
let (body, _) = read_object(&set_disks, bucket, object).await;
|
||||
assert_eq!(body, new, "the committed replacement must remain readable after the crash");
|
||||
let mut receipts = 0;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -519,6 +519,10 @@ impl SetDisks {
|
||||
let object: Arc<str> = Arc::from(object);
|
||||
let version_id: Arc<str> = Arc::from(version_id);
|
||||
let opts = *opts;
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let api_operation = rustfs_io_metrics::get_stage_metrics_enabled()
|
||||
.then(rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_context)
|
||||
.flatten();
|
||||
|
||||
let processor = runtime_sources::batch_processors().read_processor();
|
||||
let tasks: Vec<_> = disks
|
||||
@@ -530,8 +534,27 @@ impl SetDisks {
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let task_opts = opts;
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let api_operation = api_operation;
|
||||
|
||||
async move { disk.read_version(&bucket, &bucket, &object, &version_id, &task_opts).await }
|
||||
async move {
|
||||
let read_version =
|
||||
async move { disk.read_version(&bucket, &bucket, &object, &version_id, &task_opts).await };
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read_version = crate::cluster::rpc::runtime_sources::with_read_version_request_caller_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::SetDiskRead,
|
||||
read_version,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
if let Some(api_operation) = api_operation {
|
||||
rustfs_io_metrics::s3_http_metrics::with_s3_http_operation_context(Some(api_operation), read_version)
|
||||
.await
|
||||
} else {
|
||||
read_version.await
|
||||
}
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
read_version.await
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
@@ -711,34 +734,45 @@ impl SetDisks {
|
||||
let allow_read_version_coalescing = purpose == MetadataReadPurpose::GetObjectReader
|
||||
&& !crate::bucket::utils::is_meta_bucketname(bucket)
|
||||
&& crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
|
||||
let metadata_read_result = if allow_read_version_coalescing {
|
||||
Self::read_metadata_for_get_object(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Self::read_metadata_observed(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
false,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await
|
||||
let metadata_read = async {
|
||||
if allow_read_version_coalescing {
|
||||
Self::read_metadata_for_get_object(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
Self::read_metadata_observed(
|
||||
&disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
vid.as_str(),
|
||||
read_data,
|
||||
false,
|
||||
opts.incl_free_versions,
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await
|
||||
}
|
||||
};
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let metadata_read_result = crate::cluster::rpc::runtime_sources::with_read_version_caller_and_entry_if_absent(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::Object,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ObjectRead,
|
||||
metadata_read,
|
||||
)
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
let metadata_read_result = metadata_read.await;
|
||||
let metadata_read = match metadata_read_result {
|
||||
Ok(metadata_read) => metadata_read,
|
||||
Err(err) => {
|
||||
|
||||
@@ -63,6 +63,8 @@ pub(crate) mod namespace {
|
||||
pub(crate) mod object {
|
||||
use super::{Debug, Error, FileInfo, GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_storage_api::object::DeleteObjectCondition;
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
DeleteAccounting, DeletedObject, HTTPPreconditions, ObjectIO, ObjectLockDeleteOptions, ObjectLockRetentionOptions,
|
||||
ObjectOperations, ObjectPreconditionError, ObjectPreconditionPart, ObjectPreconditionState, ObjectToDelete,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -117,6 +117,47 @@ enum HealFormatPoolSkip {
|
||||
Retryable,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum HealFormatRebalance {
|
||||
Missing,
|
||||
Current(RebalanceMeta),
|
||||
Historical,
|
||||
}
|
||||
|
||||
impl HealFormatRebalance {
|
||||
fn current(&self) -> Option<&RebalanceMeta> {
|
||||
match self {
|
||||
Self::Current(meta) => Some(meta),
|
||||
Self::Missing | Self::Historical => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error(
|
||||
"format repair blocked: rebalance metadata describes {stored_pools} pools but the deployment has {current_pools}; the historical run is not confirmed terminal"
|
||||
)]
|
||||
struct HealFormatRebalanceTopologyMismatch {
|
||||
stored_pools: usize,
|
||||
current_pools: usize,
|
||||
}
|
||||
|
||||
fn classify_heal_format_rebalance(meta: Option<RebalanceMeta>, current_pools: usize) -> Result<HealFormatRebalance> {
|
||||
let Some(meta) = meta else {
|
||||
return Ok(HealFormatRebalance::Missing);
|
||||
};
|
||||
if meta.pool_stats.len() == current_pools {
|
||||
return Ok(HealFormatRebalance::Current(meta));
|
||||
}
|
||||
if meta.has_terminal_topology() {
|
||||
return Ok(HealFormatRebalance::Historical);
|
||||
}
|
||||
Err(Error::other(HealFormatRebalanceTopologyMismatch {
|
||||
stored_pools: meta.pool_stats.len(),
|
||||
current_pools,
|
||||
}))
|
||||
}
|
||||
|
||||
fn classify_heal_format_pool(
|
||||
pool_idx: usize,
|
||||
pool_cmd_line: &str,
|
||||
@@ -365,13 +406,18 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
}
|
||||
if summary.pending > 0 {
|
||||
if object.is_some() && summary.pending > 0 {
|
||||
return Err(StorageError::ErasureReadQuorum);
|
||||
}
|
||||
if object.is_some() && !opts.dry_run && summary.blocked > 0 {
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
if summary.discovered == 0 {
|
||||
let delete_recovery = rustfs_madmin::heal_commands::DeleteRecoveryStatus {
|
||||
pending: summary.pending,
|
||||
blocked: summary.blocked,
|
||||
failed: summary.failed,
|
||||
};
|
||||
if summary.discovered == 0 && !delete_recovery.has_unresolved() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(HealResultItem {
|
||||
@@ -384,14 +430,17 @@ impl ECStore {
|
||||
object: object.or(prefix).unwrap_or_default(),
|
||||
bucket,
|
||||
detail: format!(
|
||||
"delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}",
|
||||
"delete recovery{}: discovered={}, committed={}, rolled_back={}, reclaimed={}, blocked={}, pending={}, failed={}",
|
||||
if opts.dry_run { " (dry run)" } else { "" },
|
||||
summary.discovered,
|
||||
summary.committed,
|
||||
summary.rolled_back,
|
||||
summary.reclaimed,
|
||||
summary.blocked
|
||||
summary.blocked,
|
||||
summary.pending,
|
||||
summary.failed
|
||||
),
|
||||
delete_recovery: Some(delete_recovery),
|
||||
..Default::default()
|
||||
}))
|
||||
})
|
||||
@@ -479,7 +528,7 @@ impl ECStore {
|
||||
NamespaceLockGuard,
|
||||
NamespaceLockGuard,
|
||||
PoolMeta,
|
||||
Option<RebalanceMeta>,
|
||||
HealFormatRebalance,
|
||||
)> {
|
||||
let metadata_pool = self
|
||||
.pools
|
||||
@@ -535,12 +584,9 @@ impl ECStore {
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
if rebalance_meta
|
||||
.as_ref()
|
||||
.is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
|
||||
{
|
||||
return Err(heal_format_fence_lost_error());
|
||||
}
|
||||
// Keep historical status on disk, but never use its obsolete indices
|
||||
// to gate the current pools. The current pool identity remains fenced.
|
||||
let rebalance_meta = classify_heal_format_rebalance(rebalance_meta, self.pools.len())?;
|
||||
|
||||
write_state.ensure_write_safe("heal format fence failed")?;
|
||||
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
|
||||
@@ -743,7 +789,8 @@ impl ECStore {
|
||||
first_error.get_or_insert(heal_format_fence_lost_error());
|
||||
break;
|
||||
}
|
||||
if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
|
||||
if let Some(skip) =
|
||||
classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.current())
|
||||
{
|
||||
if matches!(skip, HealFormatPoolSkip::Completed) {
|
||||
count_completed += 1;
|
||||
@@ -817,6 +864,74 @@ impl ECStore {
|
||||
Ok((r, None))
|
||||
}
|
||||
|
||||
/// Read-only admission for one legacy pre-format recovery handoff. The
|
||||
/// caller holds the target execution leases and validates their owners.
|
||||
pub async fn replacement_preformat_retry_is_safe(
|
||||
&self,
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
targets: &[String],
|
||||
) -> Result<bool> {
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
|
||||
let Some(set) = pool.disk_set.get(set_index) else {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
};
|
||||
let (write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
|
||||
if !matches!(rebalance_meta, HealFormatRebalance::Historical)
|
||||
|| classify_heal_format_pool(pool_index, &pool.endpoints.cmd_line, &pool_meta, None).is_some()
|
||||
|| targets.is_empty()
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let mut slots = BTreeSet::new();
|
||||
let mut target_disks = Vec::with_capacity(targets.len());
|
||||
for target in targets {
|
||||
let Some((slot, endpoint)) = set
|
||||
.set_endpoints
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find(|(_, endpoint)| endpoint.to_string() == *target)
|
||||
else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !endpoint.is_local || !slots.insert(slot) {
|
||||
return Ok(false);
|
||||
}
|
||||
// Open the current mounted target; a cached disk handle can still
|
||||
// refer to the previous disk at the same endpoint.
|
||||
let target_disk = disk::new_disk(
|
||||
endpoint,
|
||||
&disk::DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if !target_disk.is_blank_replacement_target().await? {
|
||||
return Ok(false);
|
||||
}
|
||||
target_disks.push(target_disk);
|
||||
}
|
||||
|
||||
let (_, error) = set
|
||||
.heal_replacement_format_with_fence(true, targets, || pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost())
|
||||
.await?;
|
||||
write_state.ensure_write_safe("pre-format recovery admission fence failed")?;
|
||||
if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
|
||||
return Err(heal_format_fence_lost_error());
|
||||
}
|
||||
for target_disk in target_disks {
|
||||
if !target_disk.is_blank_replacement_target().await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(error.is_none())
|
||||
}
|
||||
|
||||
#[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
|
||||
pub async fn heal_replacement_format(
|
||||
&self,
|
||||
@@ -838,7 +953,8 @@ impl ECStore {
|
||||
})?;
|
||||
|
||||
let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
|
||||
if let Some(skip) = classify_heal_format_pool(pool_index, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref()) {
|
||||
if let Some(skip) = classify_heal_format_pool(pool_index, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.current())
|
||||
{
|
||||
return Ok((HealResultItem::default(), Some(heal_format_pool_skip_error(skip))));
|
||||
}
|
||||
|
||||
@@ -2129,9 +2245,13 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
|
||||
heal_store_with_pool_count(2).await
|
||||
}
|
||||
|
||||
async fn heal_store_with_pool_count(pool_count: usize) -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
|
||||
let mut pool_endpoints = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
for pool_index in 0..pool_count {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
@@ -2448,6 +2568,224 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
fn completed_two_pool_rebalance() -> RebalanceMeta {
|
||||
RebalanceMeta {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
pool_stats: vec![
|
||||
RebalanceStats {
|
||||
participating: true,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Completed,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
RebalanceStats::default(),
|
||||
],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn historical_rebalance_topology_requires_every_pool_to_be_terminal() {
|
||||
use crate::services::rebalance::{RebalanceStopPropagationRecord, encode_rebalance_stop_propagation_record};
|
||||
|
||||
assert!(matches!(
|
||||
classify_heal_format_rebalance(Some(completed_two_pool_rebalance()), 1),
|
||||
Ok(HealFormatRebalance::Historical)
|
||||
));
|
||||
// Expansion has the same historical-index problem as removal.
|
||||
assert!(matches!(
|
||||
classify_heal_format_rebalance(Some(completed_two_pool_rebalance()), 3),
|
||||
Ok(HealFormatRebalance::Historical)
|
||||
));
|
||||
assert!(matches!(classify_heal_format_rebalance(None, 1), Ok(HealFormatRebalance::Missing)));
|
||||
for status in [RebalStatus::Completed, RebalStatus::Stopped, RebalStatus::Failed] {
|
||||
let mut meta = completed_two_pool_rebalance();
|
||||
meta.pool_stats[0].info.status = status;
|
||||
assert!(meta.has_terminal_topology(), "terminal status {status:?}");
|
||||
}
|
||||
|
||||
let mut invalid = Vec::new();
|
||||
for index in 0..2 {
|
||||
let mut active = completed_two_pool_rebalance();
|
||||
active.stopped_at = Some(OffsetDateTime::UNIX_EPOCH);
|
||||
active.pool_stats[index].info.status = RebalStatus::Started;
|
||||
invalid.push(active);
|
||||
let mut stopping = completed_two_pool_rebalance();
|
||||
stopping.pool_stats[index].info.stopping = true;
|
||||
invalid.push(stopping);
|
||||
}
|
||||
let mut unknown_participant = completed_two_pool_rebalance();
|
||||
unknown_participant.pool_stats[0].info.status = RebalStatus::None;
|
||||
invalid.push(unknown_participant);
|
||||
let mut no_id = completed_two_pool_rebalance();
|
||||
no_id.id.clear();
|
||||
invalid.push(no_id);
|
||||
invalid.push(RebalanceMeta::default());
|
||||
for record in [
|
||||
RebalanceStopPropagationRecord {
|
||||
stop_attempt_at: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
},
|
||||
RebalanceStopPropagationRecord {
|
||||
stop_failures: vec!["peer".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
RebalanceStopPropagationRecord {
|
||||
terminal_reload_attempt_at: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
terminal_reload_failures: vec!["peer".to_string()],
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
let mut meta = completed_two_pool_rebalance();
|
||||
meta.pool_stats[1].info.last_error = Some(encode_rebalance_stop_propagation_record(&record));
|
||||
invalid.push(meta);
|
||||
}
|
||||
let mut malformed = completed_two_pool_rebalance();
|
||||
malformed.pool_stats[0].info.last_error = Some("rebalance stop propagation incomplete: invalid".to_string());
|
||||
invalid.push(malformed);
|
||||
for meta in invalid {
|
||||
let error = classify_heal_format_rebalance(Some(meta), 1).expect_err("uncertain history must block repair");
|
||||
assert!(error.to_string().contains("not confirmed terminal"));
|
||||
assert!(!error.to_string().contains("request rate"));
|
||||
}
|
||||
let mut acknowledged = completed_two_pool_rebalance();
|
||||
acknowledged.pool_stats[0].info.last_error =
|
||||
Some(encode_rebalance_stop_propagation_record(&RebalanceStopPropagationRecord {
|
||||
stop_attempt_at: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
terminal_reload_attempt_at: Some(OffsetDateTime::UNIX_EPOCH),
|
||||
..Default::default()
|
||||
}));
|
||||
assert!(acknowledged.has_terminal_topology());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn replacement_format_after_historical_pool_removal() {
|
||||
let (temp_dir, store, shutdown) = heal_store_with_pool_count(1).await;
|
||||
let mut meta = completed_two_pool_rebalance();
|
||||
meta.save(store.pools[0].clone())
|
||||
.await
|
||||
.expect("persist historical two-pool run");
|
||||
let (before, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), REBAL_META_NAME)
|
||||
.await
|
||||
.expect("read historical run before repair");
|
||||
let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
||||
|
||||
// Even a worker in a removed pool must fence every current format write.
|
||||
meta.pool_stats[1].participating = true;
|
||||
meta.pool_stats[1].info.status = RebalStatus::Started;
|
||||
meta.save(store.pools[0].clone())
|
||||
.await
|
||||
.expect("persist active historical pool");
|
||||
let err = store
|
||||
.heal_replacement_format(false, 0, 0, std::slice::from_ref(&target))
|
||||
.await
|
||||
.expect_err("removed active source must block format repair");
|
||||
assert!(err.to_string().contains("2 pools but the deployment has 1"));
|
||||
assert_heal_test_format_missing(&temp_dir, 0, 3).await;
|
||||
meta.pool_stats[1] = RebalanceStats::default();
|
||||
meta.save(store.pools[0].clone())
|
||||
.await
|
||||
.expect("persist terminal history again");
|
||||
|
||||
let (result, error) = store
|
||||
.heal_replacement_format(false, 0, 0, std::slice::from_ref(&target))
|
||||
.await
|
||||
.expect("terminal historical topology must allow target format repair");
|
||||
assert!(error.is_none(), "target repair: {error:?}");
|
||||
assert!(result.after.drives.iter().all(|drive| !drive.uuid.is_empty()));
|
||||
let disk = store.pools[0].disk_set[0].disks.read().await[3]
|
||||
.clone()
|
||||
.expect("replacement disk");
|
||||
let repaired = load_format_erasure(&disk, true).await.expect("read repaired target format");
|
||||
assert!(!repaired.erasure.this.is_nil());
|
||||
|
||||
remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
||||
let (_, error) = store
|
||||
.handle_heal_format(false)
|
||||
.await
|
||||
.expect("full format heal uses the same historical fence");
|
||||
assert!(error.is_none(), "full format repair: {error:?}");
|
||||
assert!(heal_test_format_path(&temp_dir, 0, 3).exists());
|
||||
let (after, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), REBAL_META_NAME)
|
||||
.await
|
||||
.expect("historical status remains readable");
|
||||
assert_eq!(before, after, "format repair must not rewrite historical rebalance status");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn preformat_retry_requires_historical_topology_and_a_blank_target() {
|
||||
let (temp_dir, store, shutdown) = heal_store_with_pool_count(1).await;
|
||||
completed_two_pool_rebalance()
|
||||
.save(store.pools[0].clone())
|
||||
.await
|
||||
.expect("persist historical two-pool run");
|
||||
let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
|
||||
assert!(
|
||||
!store
|
||||
.replacement_preformat_retry_is_safe(0, 0, std::slice::from_ref(&target))
|
||||
.await
|
||||
.expect("inspect target with old metadata"),
|
||||
"missing format alone must not admit a disk containing old metadata"
|
||||
);
|
||||
let target_path = temp_dir.path().join("pool0-disk3");
|
||||
for entry in std::fs::read_dir(&target_path).expect("list target fixture") {
|
||||
let entry = entry.expect("target fixture entry");
|
||||
if entry.file_type().expect("fixture entry type").is_dir() {
|
||||
std::fs::remove_dir_all(entry.path()).expect("clear replacement fixture directory");
|
||||
} else {
|
||||
std::fs::remove_file(entry.path()).expect("clear replacement fixture file");
|
||||
}
|
||||
}
|
||||
let target_disk = store.pools[0].disk_set[0].disks.read().await[3]
|
||||
.clone()
|
||||
.expect("replacement disk");
|
||||
if !cfg!(target_os = "linux") {
|
||||
assert!(target_disk.acquire_replacement_execution_lease().await.is_err());
|
||||
assert!(
|
||||
!store
|
||||
.replacement_preformat_retry_is_safe(0, 0, &[target])
|
||||
.await
|
||||
.expect("platforms without pinned mount identity must reject compatibility recovery")
|
||||
);
|
||||
shutdown.cancel();
|
||||
return;
|
||||
}
|
||||
let _lease = target_disk
|
||||
.acquire_replacement_execution_lease()
|
||||
.await
|
||||
.expect("own replacement executor");
|
||||
assert!(
|
||||
store
|
||||
.replacement_preformat_retry_is_safe(0, 0, std::slice::from_ref(&target))
|
||||
.await
|
||||
.expect("inspect blank target")
|
||||
);
|
||||
assert_heal_test_format_missing(&temp_dir, 0, 3).await;
|
||||
std::fs::create_dir(target_path.join("user-bucket")).expect("simulate residual user bucket");
|
||||
assert!(
|
||||
!store
|
||||
.replacement_preformat_retry_is_safe(0, 0, std::slice::from_ref(&target))
|
||||
.await
|
||||
.expect("inspect nonblank target")
|
||||
);
|
||||
std::fs::remove_dir(target_path.join("user-bucket")).expect("remove fixture bucket");
|
||||
delete_config(store.pools[0].clone(), REBAL_META_NAME)
|
||||
.await
|
||||
.expect("remove historical run");
|
||||
assert!(
|
||||
!store
|
||||
.replacement_preformat_retry_is_safe(0, 0, &[target])
|
||||
.await
|
||||
.expect("ordinary replacement has no compatibility authority")
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn replacement_format_heal_coexists_with_capacity_read_fence() {
|
||||
@@ -3740,7 +4078,6 @@ mod tests {
|
||||
let mut recoverable_temp_dirs = Vec::new();
|
||||
let mut recoverable_endpoints = Vec::new();
|
||||
let mut recoverable_disks = Vec::new();
|
||||
let mut unformatted_disk = None;
|
||||
for disk_index in 0..3 {
|
||||
let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
|
||||
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
|
||||
@@ -3757,15 +4094,11 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("temporary disk should open");
|
||||
if disk_index < 2 {
|
||||
let mut disk_format = recoverable_format.clone();
|
||||
disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
|
||||
save_format_file(&Some(disk.clone()), &Some(disk_format))
|
||||
.await
|
||||
.expect("recoverable format should be written");
|
||||
} else {
|
||||
unformatted_disk = Some(disk.clone());
|
||||
}
|
||||
let mut disk_format = recoverable_format.clone();
|
||||
disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
|
||||
save_format_file(&Some(disk.clone()), &Some(disk_format))
|
||||
.await
|
||||
.expect("recoverable format should be written before pool metadata");
|
||||
recoverable_temp_dirs.push(temp_dir);
|
||||
recoverable_endpoints.push(endpoint);
|
||||
recoverable_disks.push(Some(disk));
|
||||
@@ -3814,6 +4147,36 @@ mod tests {
|
||||
.expect("pool metadata should be persisted before format heal");
|
||||
*store.pool_meta_save_gate.lock().await = PoolMetaWriteState::default();
|
||||
|
||||
let replacement_endpoint = store.pools[1].endpoints.endpoints.as_ref()[2].clone();
|
||||
tokio::fs::remove_file(
|
||||
recoverable_temp_dirs[2]
|
||||
.path()
|
||||
.join(crate::disk::RUSTFS_META_BUCKET)
|
||||
.join(FORMAT_CONFIG_FILE),
|
||||
)
|
||||
.await
|
||||
.expect("the target format must be removed only after durable pool metadata exists");
|
||||
let replacement_disk = new_disk(
|
||||
&replacement_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the unformatted replacement disk must reopen");
|
||||
assert!(matches!(
|
||||
load_format_erasure(&replacement_disk, true).await,
|
||||
Err(crate::disk::error::DiskError::UnformattedDisk)
|
||||
));
|
||||
let replacement_set = &store.pools[1].disk_set[0];
|
||||
let replacement_slot = replacement_set
|
||||
.set_endpoints
|
||||
.iter()
|
||||
.position(|endpoint| endpoint == &replacement_endpoint)
|
||||
.expect("the replacement must remain in its original set");
|
||||
replacement_set.disks.write().await[replacement_slot] = Some(replacement_disk.clone());
|
||||
|
||||
let (result, err) = store
|
||||
.handle_heal_format(false)
|
||||
.await
|
||||
@@ -3823,7 +4186,7 @@ mod tests {
|
||||
"foreign format majority must not be downgraded to a successful heal: {err:?}"
|
||||
);
|
||||
assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected");
|
||||
let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true)
|
||||
let healed = load_format_erasure(&replacement_disk, true)
|
||||
.await
|
||||
.expect("the later pool should be healed despite the first pool error");
|
||||
assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
|
||||
|
||||
@@ -831,6 +831,8 @@ impl ECStore {
|
||||
}
|
||||
});
|
||||
|
||||
super::delete_recovery_runtime::start_delete_recovery(self.clone(), rx.clone());
|
||||
|
||||
runtime_sources::init_bucket_monitor_for_current_endpoints();
|
||||
crate::bucket::bucket_target_sys::BucketTargetSys::get().start_heartbeat();
|
||||
|
||||
@@ -4163,6 +4165,178 @@ mod tests {
|
||||
OfflineTestDisks { disks }
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
async fn durable_quota_writes_with_one_offline_disk(sharded: bool) {
|
||||
use crate::bucket::quota::BucketQuota;
|
||||
use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create degraded quota store dir");
|
||||
let (ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "degraded-quota", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = format!("degraded-quota-{}", Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create degraded quota bucket");
|
||||
let quota = if sharded {
|
||||
BucketQuota::new_sharded(16_384)
|
||||
} else {
|
||||
BucketQuota::new(Some(16_384))
|
||||
};
|
||||
crate::bucket::metadata_sys::update_in(
|
||||
&ctx,
|
||||
&bucket,
|
||||
crate::bucket::metadata::BUCKET_QUOTA_CONFIG_FILE,
|
||||
serde_json::to_vec("a).expect("encode durable quota"),
|
||||
)
|
||||
.await
|
||||
.expect("configure durable quota before losing a disk");
|
||||
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
assert_eq!(set.disks.read().await.len(), 4);
|
||||
let offline_disk = force_set_disk_range_offline_for_test(set, 0..1).await;
|
||||
let _without_proof = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
|
||||
assert!(crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof().is_none());
|
||||
|
||||
let payload = vec![0x51; 4096];
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
store
|
||||
.put_object(&bucket, "put", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("three healthy disks must admit a quota PUT without fleet proof");
|
||||
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, "multipart", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload with one disk offline");
|
||||
let mut reader = PutObjReader::from_vec(payload.clone());
|
||||
let part = store
|
||||
.put_object_part(&bucket, "multipart", &upload.upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart part with one disk offline");
|
||||
Arc::clone(&store)
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
"multipart",
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("three healthy disks must complete a quota multipart upload without fleet proof");
|
||||
|
||||
let source = store
|
||||
.get_object_reader(&bucket, "put", None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read degraded copy source");
|
||||
let mut copy_info = source.object_info.clone();
|
||||
let size = copy_info.get_actual_size().expect("copy source logical size");
|
||||
copy_info.put_object_reader = Some(PutObjReader::new(
|
||||
rustfs_rio::HashReader::from_stream(source.stream, size, size, None, None, false)
|
||||
.expect("construct copy source reader"),
|
||||
));
|
||||
store
|
||||
.copy_object(
|
||||
&bucket,
|
||||
"put",
|
||||
&bucket,
|
||||
"copy",
|
||||
&mut copy_info,
|
||||
&ObjectOptions::default(),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("three healthy disks must copy into a quota bucket without fleet proof");
|
||||
|
||||
// Hold the last available credit before publication so the competing
|
||||
// writer must account for an outstanding reservation, not just objects.
|
||||
let barrier = PutObjectCommitBarrier::install(&bucket, "reserved", PutObjectCommitPause::BeforeQuotaRename);
|
||||
let first_store = Arc::clone(&store);
|
||||
let first_bucket = bucket.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x52; 4096]);
|
||||
first_store
|
||||
.put_object(&first_bucket, "reserved", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("the last within-limit writer must reach its commit barrier");
|
||||
let mut reader = PutObjReader::from_vec(vec![0x53; 4096]);
|
||||
let denied = store
|
||||
.put_object(&bucket, "over-limit", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("an outstanding reservation must prevent concurrent quota oversubscription");
|
||||
assert!(matches!(denied, StorageError::QuotaExceeded { limit: 16_384, .. }));
|
||||
barrier.release();
|
||||
first
|
||||
.await
|
||||
.expect("reserved PUT task should not panic")
|
||||
.expect("the reserved writer must commit while one disk is offline");
|
||||
|
||||
for object in ["put", "multipart", "copy", "reserved"] {
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("degraded quota writes must be readable");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("read committed quota object");
|
||||
let expected = if object == "reserved" {
|
||||
vec![0x52; 4096]
|
||||
} else {
|
||||
payload.clone()
|
||||
};
|
||||
assert_eq!(body, expected, "committed body for {object}");
|
||||
}
|
||||
store
|
||||
.get_object_info(&bucket, "over-limit", &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the over-limit writer must not publish an object");
|
||||
|
||||
let second_offline_disk = force_set_disk_range_offline_for_test(set, 1..2).await;
|
||||
let mut reader = PutObjReader::from_vec(vec![0x54; 4096]);
|
||||
let denied = store
|
||||
.put_object(&bucket, "put", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("a zero-growth overwrite must still fail when the actual write quorum is lost");
|
||||
assert!(
|
||||
matches!(
|
||||
denied,
|
||||
StorageError::ErasureWriteQuorum
|
||||
| StorageError::ErasureReadQuorum
|
||||
| StorageError::InsufficientWriteQuorum(_, _)
|
||||
| StorageError::InsufficientReadQuorum(_, _)
|
||||
),
|
||||
"loss of actual disk quorum must return a storage quorum error, got {denied:?}"
|
||||
);
|
||||
shutdown.cancel();
|
||||
drop(second_offline_disk);
|
||||
drop(offline_disk);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn durable_quota_v1_writes_with_one_offline_disk_and_no_fleet_proof() {
|
||||
run_large_stack_async_test("degraded-quota-v1", || durable_quota_writes_with_one_offline_disk(false));
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn durable_quota_v2_writes_with_one_offline_disk_and_no_fleet_proof() {
|
||||
run_large_stack_async_test("degraded-quota-v2", || durable_quota_writes_with_one_offline_disk(true));
|
||||
}
|
||||
|
||||
fn active_rebalance_meta_for_pool(pool_count: usize, active_pool_idx: usize) -> RebalanceMeta {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let mut pool_stats = vec![RebalanceStats::default(); pool_count];
|
||||
@@ -5030,6 +5204,339 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn repeated_abort_recovers_exact_orphan_before_decommission_drains() {
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use sha2::Digest;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create orphan recovery store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "orphan-multipart-drain", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-multipart-{}", uuid::Uuid::new_v4());
|
||||
let object = "exact-object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation");
|
||||
let lifecycle_guard = store.acquire_bucket_lifecycle_read_lock(&bucket).await.expect("bucket guard");
|
||||
let mut upload_opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &upload_opts)
|
||||
.await
|
||||
.expect("create source upload");
|
||||
drop(lifecycle_guard);
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial abort");
|
||||
|
||||
let upload_sha =
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
let staging_dir = |pool_index: usize, disk_index: usize| {
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool{pool_index}/set0/disk{disk_index}"))
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid)
|
||||
};
|
||||
for disk_index in 0..3 {
|
||||
let path = staging_dir(0, disk_index);
|
||||
tokio::fs::create_dir_all(&path).await.expect("create late part directory");
|
||||
tokio::fs::write(path.join("part.1"), b"late part after abort")
|
||||
.await
|
||||
.expect("write late part fixture");
|
||||
}
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("logical upload listing")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
assert!(store.ensure_decommission_multipart_uploads_drained_for_test(0).await.is_err());
|
||||
assert!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, "different-key", &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"the same UploadID for a different key must not recover this path"
|
||||
);
|
||||
|
||||
// A minority metadata copy in another pool blocks recovery everywhere.
|
||||
let conflicting_metadata = staging_dir(1, 0).join(crate::disk::STORAGE_FORMAT_FILE);
|
||||
tokio::fs::create_dir_all(conflicting_metadata.parent().expect("metadata parent"))
|
||||
.await
|
||||
.expect("create conflicting pool staging");
|
||||
tokio::fs::write(&conflicting_metadata, b"unreadable metadata")
|
||||
.await
|
||||
.expect("write conflicting metadata");
|
||||
assert!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"no pool may be cleaned until every pool proves absence"
|
||||
);
|
||||
for disk_index in 0..3 {
|
||||
assert!(staging_dir(0, disk_index).join("part.1").exists());
|
||||
}
|
||||
tokio::fs::remove_file(conflicting_metadata)
|
||||
.await
|
||||
.expect("remove conflicting fixture");
|
||||
let (paused, release) = crate::store::multipart::pause_orphan_multipart_cleanup_for_test(&bucket);
|
||||
let recovery_store = store.clone();
|
||||
let recovery_bucket = bucket.clone();
|
||||
let recovery_upload = upload.upload_id.clone();
|
||||
let mut recovery = tokio::spawn(async move {
|
||||
recovery_store
|
||||
.abort_multipart_upload(&recovery_bucket, object, &recovery_upload, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
paused = paused => paused.expect("cleanup owner announces arrival"),
|
||||
result = &mut recovery => panic!("recovery returned before physical cleanup: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("recovery should reach physical cleanup");
|
||||
recovery.abort();
|
||||
assert!(recovery.await.expect_err("request waiter is cancellable").is_cancelled());
|
||||
let upload_lock = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_MULTIPART_BUCKET, &format!("{upload_sha}/{upload_uuid}"))
|
||||
.await
|
||||
.expect("same upload namespace");
|
||||
let mut upload_writer = Box::pin(upload_lock.get_write_lock(Duration::from_secs(10)));
|
||||
let mut bucket_writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
|
||||
assert!(
|
||||
futures::poll!(&mut upload_writer).is_pending(),
|
||||
"cancelled recovery must retain the upload lock"
|
||||
);
|
||||
assert!(
|
||||
futures::poll!(&mut bucket_writer).is_pending(),
|
||||
"cancelled recovery must retain the real bucket guard"
|
||||
);
|
||||
release.send(()).expect("allow physical cleanup to finish");
|
||||
let (upload_guard, bucket_guard) =
|
||||
tokio::time::timeout(Duration::from_secs(10), async { tokio::join!(upload_writer, bucket_writer) })
|
||||
.await
|
||||
.expect("both guards should release after physical cleanup");
|
||||
drop(upload_guard.expect("upload writer resumes"));
|
||||
drop(bucket_guard.expect("bucket writer resumes"));
|
||||
for pool_index in 0..2 {
|
||||
for disk_index in 0..4 {
|
||||
assert!(!staging_dir(pool_index, disk_index).exists(), "every checked staging path must be absent");
|
||||
}
|
||||
}
|
||||
store
|
||||
.ensure_decommission_multipart_uploads_drained_for_test(0)
|
||||
.await
|
||||
.expect("physical decommission gate should open after recovery");
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::InvalidUploadID(_, _, _))
|
||||
),
|
||||
"an absent upload must not claim to have recovered anything"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn repeated_abort_keeps_no_such_upload_while_a_disk_is_offline() {
|
||||
use sha2::Digest;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create offline-disk recovery store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "orphan-multipart-offline", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-multipart-{}", uuid::Uuid::new_v4());
|
||||
let object = "offline-object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation");
|
||||
let lifecycle_guard = store.acquire_bucket_lifecycle_read_lock(&bucket).await.expect("bucket guard");
|
||||
let mut upload_opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &upload_opts)
|
||||
.await
|
||||
.expect("create source upload");
|
||||
drop(lifecycle_guard);
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial abort");
|
||||
|
||||
let upload_sha =
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
let staging_dir = |disk_index: usize| {
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool0/set0/disk{disk_index}"))
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid)
|
||||
};
|
||||
for disk_index in 0..3 {
|
||||
let path = staging_dir(disk_index);
|
||||
tokio::fs::create_dir_all(&path).await.expect("create late part directory");
|
||||
tokio::fs::write(path.join("part.1"), b"late part after abort")
|
||||
.await
|
||||
.expect("write late part fixture");
|
||||
}
|
||||
|
||||
// One offline disk means absence cannot be proven. The idempotent Abort
|
||||
// must keep answering NoSuchUpload instead of surfacing a disk error,
|
||||
// and must not delete anything.
|
||||
let set = store.pools[0].get_disks_by_key(object);
|
||||
let offline = set.disks.write().await[3].take();
|
||||
assert!(offline.is_some(), "fixture disk must have been online");
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::InvalidUploadID(_, _, _))
|
||||
),
|
||||
"unprovable absence must keep NoSuchUpload while a disk is offline"
|
||||
);
|
||||
for disk_index in 0..3 {
|
||||
assert!(
|
||||
staging_dir(disk_index).join("part.1").exists(),
|
||||
"no staging may be deleted while a disk is offline"
|
||||
);
|
||||
}
|
||||
|
||||
set.disks.write().await[3] = offline;
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recovery resumes once every disk is observable");
|
||||
for disk_index in 0..4 {
|
||||
assert!(!staging_dir(disk_index).exists(), "recovered staging must be absent");
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn orphan_multipart_recovery_preserves_current_bucket_incarnation() {
|
||||
use sha2::Digest;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create bucket lifetime recovery store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "orphan-bucket-lifetime", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-lifetime-{}", uuid::Uuid::new_v4());
|
||||
let object = "object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original bucket");
|
||||
let original_incarnation = store.bucket_incarnation_id(&bucket).await.expect("original incarnation");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create original upload");
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("abort original upload");
|
||||
store
|
||||
.delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
assert_ne!(store.bucket_incarnation_id(&bucket).await.expect("new incarnation"), original_incarnation);
|
||||
let mut reader = PutObjReader::from_vec(b"current bucket object".to_vec());
|
||||
store
|
||||
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write current bucket object");
|
||||
let current_upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create current incarnation upload for the same key");
|
||||
|
||||
let upload_sha =
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
let leftover = temp_dir
|
||||
.path()
|
||||
.join("pool0/set0/disk0")
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(upload_sha)
|
||||
.join(upload_uuid)
|
||||
.join("part.1");
|
||||
tokio::fs::create_dir_all(leftover.parent().expect("staging parent"))
|
||||
.await
|
||||
.expect("create old incarnation staging");
|
||||
tokio::fs::write(&leftover, b"previous bucket lifetime")
|
||||
.await
|
||||
.expect("write previous incarnation fixture");
|
||||
for opts in [
|
||||
ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(original_incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(uuid::Uuid::nil()),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &opts)
|
||||
.await
|
||||
.is_err(),
|
||||
"a stale or nil current-bucket identity must fail closed"
|
||||
);
|
||||
assert!(leftover.exists(), "failed bucket lifetime proof must preserve the path");
|
||||
}
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("exact metadata-less staging from an old incarnation is unreferenced");
|
||||
assert!(!leftover.exists());
|
||||
store
|
||||
.get_multipart_info(&bucket, object, ¤t_upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recovery must preserve the current incarnation's active upload");
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("current object must remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("read current object");
|
||||
assert_eq!(body, b"current bucket object");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn active_multipart_upload_routes_before_faulted_suspended_source() {
|
||||
@@ -5181,6 +5688,179 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn decommission_listing_accepts_committed_inline_layout_change() {
|
||||
// Keep the native migration scenario off the small debug test stack.
|
||||
std::thread::Builder::new()
|
||||
.stack_size(16 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.worker_threads(2)
|
||||
.build()
|
||||
.expect("build migration listing runtime");
|
||||
runtime.block_on(async move {
|
||||
use crate::config::storageclass::{
|
||||
INLINE_BLOCK_ENV, OPTIMIZE_ENV, RRS_ENV, STANDARD_ENV, lookup_config_for_pools,
|
||||
};
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create migration listing store");
|
||||
let (_ctx, store, shutdown) = Box::pin(without_storage_class_env(build_isolated_test_store(
|
||||
temp_dir.path(),
|
||||
"decommission-inline-listing",
|
||||
&[3, 9],
|
||||
)))
|
||||
.await;
|
||||
let _shutdown_guard = shutdown.drop_guard();
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "decommission-inline-listing";
|
||||
let object = "object.bin";
|
||||
let payload = vec![0x5a; 64 * 1024];
|
||||
Box::pin(store.make_bucket(bucket, &MakeBucketOptions::default()))
|
||||
.await
|
||||
.expect("create migration listing bucket");
|
||||
|
||||
// A source written with inline storage disabled remains valid after
|
||||
// the destination starts using the default inline policy.
|
||||
let external_config = temp_env::with_vars(
|
||||
[
|
||||
(STANDARD_ENV, None),
|
||||
(RRS_ENV, None),
|
||||
(OPTIMIZE_ENV, None),
|
||||
(INLINE_BLOCK_ENV, Some("0")),
|
||||
],
|
||||
|| lookup_config_for_pools(&KVS::new(), &[3, 9]),
|
||||
)
|
||||
.expect("external storage policy");
|
||||
for set in &store.pools[1].disk_set {
|
||||
set.set_test_storage_class_config(external_config.clone());
|
||||
}
|
||||
let source = Box::pin(store.pools[1].put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(payload.clone()),
|
||||
&ObjectOptions::default(),
|
||||
))
|
||||
.await
|
||||
.expect("write external source");
|
||||
assert!(!source.inlined, "fixture must use external source shards");
|
||||
store.pool_meta.write().await.pools[1].decommission = Some(PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let reader = Box::pin(store.pools[1].get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
data_movement: true,
|
||||
raw_data_movement_read: true,
|
||||
..Default::default()
|
||||
},
|
||||
))
|
||||
.await
|
||||
.expect("read migration source");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
Box::pin(crate::data_movement::migrate_decommission_object(
|
||||
store.clone(),
|
||||
1,
|
||||
bucket.to_string(),
|
||||
reader,
|
||||
Some(incarnation),
|
||||
"test_decommission_inline_listing",
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.expect("commit migrated target while retaining source");
|
||||
|
||||
for pool_index in [0, 1] {
|
||||
let mut reader = Box::pin(store.pools[pool_index].get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions::default(),
|
||||
))
|
||||
.await
|
||||
.expect("both physical copies must remain readable");
|
||||
assert_eq!(reader.object_info.inlined, pool_index == 0);
|
||||
assert_eq!(reader.object_info.etag, source.etag);
|
||||
assert_eq!(reader.object_info.mod_time, source.mod_time);
|
||||
let mut actual = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("read complete object body");
|
||||
assert_eq!(actual, payload);
|
||||
}
|
||||
|
||||
Box::pin(store.pools[0].put_object(
|
||||
bucket,
|
||||
"z-sentinel",
|
||||
&mut PutObjReader::from_vec(b"sentinel".to_vec()),
|
||||
&ObjectOptions::default(),
|
||||
))
|
||||
.await
|
||||
.expect("write pagination sentinel");
|
||||
let listing = Box::pin(
|
||||
store
|
||||
.clone()
|
||||
.inner_list_objects_v2(bucket, "", None, None, 1, false, None, false),
|
||||
)
|
||||
.await
|
||||
.expect("listing must merge committed source and target copies");
|
||||
assert!(listing.is_truncated);
|
||||
assert_eq!(listing.objects.len(), 1);
|
||||
assert_eq!(listing.objects[0].name, object);
|
||||
assert_eq!(listing.objects[0].etag, source.etag);
|
||||
assert_eq!(listing.objects[0].size, i64::try_from(payload.len()).expect("payload size"));
|
||||
let next = Box::pin(store.clone().inner_list_objects_v2(
|
||||
bucket,
|
||||
"",
|
||||
listing.next_continuation_token,
|
||||
None,
|
||||
1,
|
||||
false,
|
||||
None,
|
||||
false,
|
||||
))
|
||||
.await
|
||||
.expect("continue after the merged object");
|
||||
assert!(!next.is_truncated);
|
||||
assert_eq!(next.objects.len(), 1);
|
||||
assert_eq!(next.objects[0].name, "z-sentinel");
|
||||
|
||||
Box::pin(store.pools[1].delete_object(bucket, object, ObjectOptions::default()))
|
||||
.await
|
||||
.expect("remove the migrated source copy");
|
||||
let after_cleanup = Box::pin(
|
||||
store
|
||||
.clone()
|
||||
.inner_list_objects_v2(bucket, "", None, None, 100, false, None, false),
|
||||
)
|
||||
.await
|
||||
.expect("listing must remain complete after source cleanup");
|
||||
assert!(!after_cleanup.is_truncated);
|
||||
assert_eq!(
|
||||
after_cleanup
|
||||
.objects
|
||||
.iter()
|
||||
.map(|info| info.name.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[object, "z-sentinel"]
|
||||
);
|
||||
});
|
||||
})
|
||||
.expect("spawn migration listing test")
|
||||
.join()
|
||||
.expect("migration listing test must complete");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
fn data_movement_conflicts_and_multipart_retries_converge_safely() {
|
||||
@@ -16495,11 +17175,10 @@ mod tests {
|
||||
.delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("one post-commit error plus one offline disk must miss delete quorum");
|
||||
assert!(matches!(err, StorageError::Io(_)), "unexpected local delete failure: {err:?}");
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("Storage resources are insufficient for the write operation"),
|
||||
"the caller must observe the wrapped quorum failure: {err}"
|
||||
assert_eq!(
|
||||
err,
|
||||
StorageError::InsufficientWriteQuorum(bucket.to_owned(), object.to_owned()),
|
||||
"the caller must observe the typed quorum failure with its object identity"
|
||||
);
|
||||
set.disks.write().await[1] = Some(saved_disk);
|
||||
|
||||
@@ -18591,6 +19270,70 @@ mod tests {
|
||||
assert_eq!(body, b"new-generation");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial_test::serial]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn cancelled_ecstore_put_part_keeps_bucket_lifecycle_lock_through_commit() {
|
||||
for pause in [
|
||||
crate::set_disk::MultipartCommitPause::PutPartAfterCapacityAdmission,
|
||||
crate::set_disk::MultipartCommitPause::PutPartAfterRename,
|
||||
] {
|
||||
let temp_dir = tempfile::tempdir().expect("create cancelled multipart lifecycle store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "cancelled-multipart-lifecycle", &[4, 4]))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = format!("cancelled-part-lifecycle-{}", Uuid::new_v4());
|
||||
let object = "part.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for multipart lifecycle cancellation");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload through the owning ECStore");
|
||||
let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause);
|
||||
let put_store = Arc::clone(&store);
|
||||
let put_bucket = bucket.clone();
|
||||
let put_upload_id = upload.upload_id.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x45; 4096]);
|
||||
put_store
|
||||
.put_object_part(&put_bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
put.abort();
|
||||
assert!(
|
||||
put.await
|
||||
.expect_err("the ECStore request waiter must remain cancellable")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let mut writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut writer).await.is_err(),
|
||||
"canceling UploadPart at {pause:?} must not release its bucket lifecycle owner before the commit drains"
|
||||
);
|
||||
barrier.release();
|
||||
let lifecycle_guard = tokio::time::timeout(Duration::from_secs(10), writer)
|
||||
.await
|
||||
.expect("the bucket lifecycle writer must resume after the part commit drains")
|
||||
.expect("acquire the bucket lifecycle write lock after cancellation");
|
||||
drop(lifecycle_guard);
|
||||
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the cancelled request's owned part commit must finish successfully");
|
||||
assert_eq!(parts.parts.len(), 1, "the commit must publish exactly the uploaded part at {pause:?}");
|
||||
assert_eq!(parts.parts[0].part_num, 1);
|
||||
assert_eq!(parts.parts[0].size, 4096);
|
||||
shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn pre_upgrade_multipart_upload_can_complete_and_abort_in_the_same_bucket_lifetime() {
|
||||
|
||||
@@ -56,7 +56,7 @@ use rustfs_filemeta::{
|
||||
};
|
||||
use rustfs_io_metrics::{
|
||||
LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED, LIST_OBJECTS_SOURCE_WALKER,
|
||||
ListObjectsGatherObservation, ListObjectsIndexPageObservation,
|
||||
ListObjectsGatherCaller, ListObjectsGatherObservation, ListObjectsIndexPageObservation,
|
||||
};
|
||||
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
|
||||
use std::cmp::Reverse;
|
||||
@@ -109,6 +109,7 @@ fn recursive_walk_list_path_options(bucket: String, prefix: &str, opts: &WalkOpt
|
||||
incl_deleted: true,
|
||||
recursive: true,
|
||||
versioned: true,
|
||||
caller: ListObjectsGatherCaller::S3ListVersions,
|
||||
walkdir_timeout: opts.walkdir_timeout,
|
||||
walkdir_stall_timeout: opts.walkdir_stall_timeout,
|
||||
..Default::default()
|
||||
@@ -274,6 +275,7 @@ fn log_list_path_finished(
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct ListPathOptions {
|
||||
pub caller: ListObjectsGatherCaller,
|
||||
pub id: Option<String>,
|
||||
|
||||
// Bucket of the listing.
|
||||
@@ -4116,6 +4118,7 @@ impl ECStore {
|
||||
let effective_max_keys = if max_keys <= 0 { 0 } else { max_keys_plus_one(max_keys, true) };
|
||||
let mut opts = ListPathOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
caller: ListObjectsGatherCaller::S3List,
|
||||
prefix: prefix.to_owned(),
|
||||
separator: delimiter.clone(),
|
||||
// Always request max_keys + 1 to detect if there are more results
|
||||
@@ -4261,6 +4264,79 @@ impl ECStore {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Capture one logical key's version group for write-triggered expiry.
|
||||
pub(crate) async fn get_object_versions_for_lifecycle(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
bucket_incarnation_id: Uuid,
|
||||
) -> Result<Vec<ObjectInfo>> {
|
||||
crate::bucket::utils::check_object_args(bucket, object)?;
|
||||
let mut opts = opts.clone();
|
||||
// Lock order matches writes: bucket lifecycle, metadata validation,
|
||||
// then object namespace. Borrow fences supplied by an outer operation.
|
||||
let bucket_guard = if opts.bucket_lifecycle_lock_fence.is_none() {
|
||||
let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
opts.add_bucket_lifecycle_lock_guard(&guard);
|
||||
Some(guard)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let incarnation = crate::bucket::metadata_sys::get_bucket_incarnation_id_for_options_in(&self.ctx, bucket, &opts).await?;
|
||||
if incarnation != bucket_incarnation_id {
|
||||
return Err(Error::BucketNotFound(bucket.to_owned()));
|
||||
}
|
||||
let metadata_sys = crate::bucket::metadata_sys::bucket_metadata_sys_of(&self.ctx)?;
|
||||
let metadata_sys = metadata_sys.read().await.clone();
|
||||
let (versioning, _) = metadata_sys.get_versioning_config(bucket).await?;
|
||||
let disk_object = path::encode_dir_object(object);
|
||||
let object_guard = self
|
||||
.acquire_object_read_lock_if_needed("lifecycle_versions", bucket, &disk_object, &mut opts)
|
||||
.await?;
|
||||
|
||||
// As with version listings, include every physical set: migration can
|
||||
// leave a key's history in more than its current hash destination.
|
||||
let reads = self
|
||||
.pools
|
||||
.iter()
|
||||
.flat_map(|pool| &pool.disk_set)
|
||||
.map(|set| set.read_object_versions_for_lifecycle(bucket, object));
|
||||
let mut entries = Vec::new();
|
||||
for result in join_all(reads).await {
|
||||
if let Some(entry) = result.map_err(|error| to_object_err(error.into(), vec![bucket, object]))? {
|
||||
entries.push(entry);
|
||||
}
|
||||
}
|
||||
if self.pools.is_empty()
|
||||
|| bucket_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| opts
|
||||
.bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(|fence| fence.is_lock_lost())
|
||||
|| opts.namespace_lock_fence.as_ref().is_some_and(|fence| fence.is_lock_lost())
|
||||
|| object_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
let mut entries = entries.into_iter();
|
||||
let Some(mut entry) = entries.next() else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
if entries.len() > 0 {
|
||||
merge_object_entry_versions(&mut entry, entries)?;
|
||||
}
|
||||
// This projection excludes tier free-version bookkeeping but retains
|
||||
// pending version purges for the evaluator's replication safeguards.
|
||||
let versions = entry.file_info_versions(bucket)?;
|
||||
let versioned = versioning.versioned(object);
|
||||
Ok(versions
|
||||
.versions
|
||||
.iter()
|
||||
.map(|version| ObjectInfo::from_file_info(version, bucket, object, versioned))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn inner_list_object_versions_with_projection(
|
||||
self: Arc<Self>,
|
||||
input: ListObjectVersionsInput<'_>,
|
||||
@@ -4289,6 +4365,11 @@ impl ECStore {
|
||||
let effective_max_keys = list_versions_scan_limit(max_keys, has_version_marker);
|
||||
let mut opts = ListPathOptions {
|
||||
bucket: bucket.to_owned(),
|
||||
caller: if include_version_purge {
|
||||
ListObjectsGatherCaller::Lifecycle
|
||||
} else {
|
||||
ListObjectsGatherCaller::S3ListVersions
|
||||
},
|
||||
prefix: prefix.to_owned(),
|
||||
separator: delimiter.clone(),
|
||||
limit: effective_max_keys,
|
||||
@@ -4993,11 +5074,45 @@ where
|
||||
let _cancel_guard = cancel.clone().drop_guard();
|
||||
let (result_tx, mut result_rx) = mpsc::channel(1);
|
||||
let mut workers = JoinSet::new();
|
||||
workers.spawn(async move { producer.await.map(|()| None) }.instrument(tracing::Span::current()));
|
||||
let gather_cancel = cancel.clone();
|
||||
workers.spawn(
|
||||
async move { gather_results(gather_cancel, opts, recv, result_tx).await.map(Some) }.instrument(tracing::Span::current()),
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let mut diagnostic_context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::ListObjects,
|
||||
);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let request_caller = diagnostic_context.caller.or_else(|| {
|
||||
use rustfs_io_metrics::ListObjectsGatherCaller;
|
||||
use rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller;
|
||||
|
||||
match opts.caller {
|
||||
ListObjectsGatherCaller::S3List | ListObjectsGatherCaller::S3ListVersions => {
|
||||
Some(ReadVersionRequestCaller::ListObjects)
|
||||
}
|
||||
ListObjectsGatherCaller::Lifecycle => Some(ReadVersionRequestCaller::Lifecycle),
|
||||
ListObjectsGatherCaller::Other => Some(ReadVersionRequestCaller::Other),
|
||||
ListObjectsGatherCaller::Unattributed => None,
|
||||
}
|
||||
});
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
{
|
||||
diagnostic_context.caller = request_caller;
|
||||
}
|
||||
let producer = async move { producer.await.map(|()| None) };
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let producer = crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
diagnostic_context,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::ListProducer,
|
||||
producer,
|
||||
);
|
||||
workers.spawn(producer.instrument(tracing::Span::current()));
|
||||
let gather_cancel = cancel.clone();
|
||||
let gather = async move { gather_results(gather_cancel, opts, recv, result_tx).await.map(Some) };
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let gather = crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
diagnostic_context,
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionContextBoundary::ListGather,
|
||||
gather,
|
||||
);
|
||||
workers.spawn(gather.instrument(tracing::Span::current()));
|
||||
|
||||
let mut failure = None;
|
||||
let mut stopping = false;
|
||||
@@ -5133,6 +5248,7 @@ async fn gather_results(
|
||||
let duration_ms = started.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_list_objects_gather(ListObjectsGatherObservation {
|
||||
source: LIST_OBJECTS_SOURCE_WALKER,
|
||||
caller: opts.caller,
|
||||
outcome: LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED,
|
||||
limit: opts.limit,
|
||||
scanned_entries,
|
||||
@@ -5195,6 +5311,7 @@ async fn gather_results(
|
||||
let duration_ms = started.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_list_objects_gather(ListObjectsGatherObservation {
|
||||
source: LIST_OBJECTS_SOURCE_WALKER,
|
||||
caller: opts.caller,
|
||||
outcome: LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED,
|
||||
limit: opts.limit,
|
||||
scanned_entries,
|
||||
@@ -5326,6 +5443,23 @@ async fn send_or_cancel(rx: &CancellationToken, out_channel: &Sender<MetaCacheEn
|
||||
}
|
||||
}
|
||||
|
||||
fn listing_merge_corruption(object: &str, version_id: Option<Uuid>, validation: &'static str) -> Error {
|
||||
const EVENT_LISTING_MERGE_REJECTED: &str = "listing_merge_rejected";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_LISTING: &str = "listing";
|
||||
error!(
|
||||
event = EVENT_LISTING_MERGE_REJECTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LISTING,
|
||||
result = "rejected",
|
||||
object,
|
||||
version_id = ?version_id,
|
||||
validation,
|
||||
"Listing version merge rejected metadata"
|
||||
);
|
||||
Error::FileCorrupt
|
||||
}
|
||||
|
||||
/// Each input has already been resolved inside its own erasure set. This is a
|
||||
/// union of version histories, never a quorum vote between unrelated pools.
|
||||
fn merge_object_entry_versions(first: &mut MetaCacheEntry, others: impl Iterator<Item = MetaCacheEntry>) -> Result<()> {
|
||||
@@ -5334,23 +5468,26 @@ fn merge_object_entry_versions(first: &mut MetaCacheEntry, others: impl Iterator
|
||||
for mut entry in std::iter::once(std::mem::take(first)).chain(others) {
|
||||
let meta = match entry.cached.take() {
|
||||
Some(meta) => meta,
|
||||
None => FileMeta::load(&entry.metadata).map_err(|_| Error::FileCorrupt)?,
|
||||
None => FileMeta::load(&entry.metadata).map_err(|_| listing_merge_corruption(&name, None, "metadata_decode"))?,
|
||||
};
|
||||
if meta.versions.is_empty() {
|
||||
return Err(Error::FileCorrupt);
|
||||
return Err(listing_merge_corruption(&name, None, "empty_version_stack"));
|
||||
}
|
||||
for version in meta.versions {
|
||||
let parsed = version.parse_version_meta().map_err(|_| Error::FileCorrupt)?;
|
||||
let corrupt = |validation| listing_merge_corruption(&name, version.header.version_id, validation);
|
||||
let parsed = version.parse_version_meta().map_err(|_| corrupt("version_decode"))?;
|
||||
if !parsed.valid() || parsed.version_type != version.header.version_type {
|
||||
return Err(Error::FileCorrupt);
|
||||
return Err(corrupt("version_type_or_validity"));
|
||||
}
|
||||
let fi = parsed.into_fileinfo("", &name, true).map_err(|_| Error::FileCorrupt)?;
|
||||
let fi = parsed
|
||||
.into_fileinfo("", &name, true)
|
||||
.map_err(|_| corrupt("file_info_decode"))?;
|
||||
let version_id = fi.version_id.filter(|id| !id.is_nil());
|
||||
if version_id != version.header.version_id.filter(|id| !id.is_nil())
|
||||
|| fi.mod_time != version.header.mod_time
|
||||
|| fi.tier_free_version() != version.header.free_version()
|
||||
{
|
||||
return Err(Error::FileCorrupt);
|
||||
return Err(corrupt("version_header_mismatch"));
|
||||
}
|
||||
let info = ObjectInfo::from_file_info(&fi, "", &name, true);
|
||||
let identity = (version_id, version.header.free_version());
|
||||
@@ -5374,7 +5511,7 @@ fn merge_object_entry_versions(first: &mut MetaCacheEntry, others: impl Iterator
|
||||
crate::data_movement::is_equivalent_data_movement_object_identity(&info, previous_info, true, true)
|
||||
};
|
||||
if !equivalent {
|
||||
return Err(Error::FileCorrupt);
|
||||
return Err(corrupt("version_identity_conflict"));
|
||||
}
|
||||
// Equivalent migrated copies can have different coding or
|
||||
// data directories. Choose a stable representation without
|
||||
@@ -7229,6 +7366,9 @@ fn calc_common_counter(infos: &[DiskInfo], read_quorum: usize) -> u64 {
|
||||
#[cfg(test)]
|
||||
mod differential_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod lifecycle_exact_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::{
|
||||
@@ -7458,7 +7598,7 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
fn test_pending_version_purge_meta_entry(bucket: &str, name: &str) -> (MetaCacheEntry, Uuid) {
|
||||
pub(super) fn test_pending_version_purge_meta_entry(bucket: &str, name: &str) -> (MetaCacheEntry, Uuid) {
|
||||
let version_id = Uuid::from_u128(0x1234_5678_9abc_def0_1234_5678_9abc_def0);
|
||||
let data_dir = Uuid::from_u128(0xabcd_ef12_3456_7890_abcd_ef12_3456_7890);
|
||||
let mod_time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
|
||||
@@ -12117,6 +12257,82 @@ mod test {
|
||||
assert_eq!(versions.versions[0].metadata.get("etag").map(String::as_str), Some("same-etag"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn merge_entry_channels_data_movement_accepts_inline_layout_changes() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
|
||||
let external = test_object_meta_entry_with_erasure_versions("key", &[(time, "same-etag", 2, 1)]);
|
||||
let inline = rewrite_test_version(
|
||||
test_object_meta_entry_with_erasure_versions("key", &[(time, "same-etag", 5, 4)]),
|
||||
|version| {
|
||||
rustfs_utils::http::insert_bytes(
|
||||
&mut version.object.as_mut().expect("object version").meta_sys,
|
||||
rustfs_utils::http::SUFFIX_INLINE_DATA,
|
||||
b"true".to_vec(),
|
||||
);
|
||||
},
|
||||
);
|
||||
let legacy_inline = rewrite_test_version(inline.clone(), |version| {
|
||||
version
|
||||
.object
|
||||
.as_mut()
|
||||
.expect("object version")
|
||||
.meta_sys
|
||||
.remove(&rustfs_utils::http::internal_key_rustfs(rustfs_utils::http::SUFFIX_INLINE_DATA));
|
||||
});
|
||||
let copies = [external, legacy_inline, inline];
|
||||
let mut expected = None;
|
||||
for order in [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]] {
|
||||
let merged = merge_test_object_entries(order.map(|index| copies[index].clone()).to_vec())
|
||||
.await
|
||||
.expect("physical inline placement and legacy aliases must not change object identity");
|
||||
assert_eq!(
|
||||
merged
|
||||
.file_info_versions("bucket")
|
||||
.expect("decode merged history")
|
||||
.versions
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
if let Some(expected) = &expected {
|
||||
assert_eq!(&merged.metadata, expected, "representation must not depend on pool order");
|
||||
} else {
|
||||
expected = Some(merged.metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_movement_listing_conflict_logs_identity_without_metadata_values() {
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.finish();
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid timestamp");
|
||||
let mut original = test_object_meta_entry_with_erasure_versions("failing-key", &[(time, "same-etag", 2, 1)]);
|
||||
let conflicting = rewrite_test_version(original.clone(), |version| {
|
||||
version
|
||||
.object
|
||||
.as_mut()
|
||||
.expect("object version")
|
||||
.meta_user
|
||||
.insert("x-amz-meta-private".to_string(), "sensitive-value-must-not-be-logged".to_string());
|
||||
});
|
||||
assert_eq!(
|
||||
super::merge_object_entry_versions(&mut original, std::iter::once(conflicting))
|
||||
.expect_err("conflicting user metadata must still fail closed"),
|
||||
StorageError::FileCorrupt
|
||||
);
|
||||
let output = logs.contents();
|
||||
assert!(output.contains("listing_merge_rejected"));
|
||||
assert!(output.contains("failing-key"));
|
||||
assert!(output.contains("00000000-0000-0000-0000-000000000001"));
|
||||
assert!(output.contains("version_identity_conflict"));
|
||||
assert!(!output.contains("sensitive-value-must-not-be-logged"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn merge_entry_channels_defers_free_version_while_same_remote_source_is_live() {
|
||||
let live = test_transitioned_meta_entry("key", "remote/shared", false);
|
||||
@@ -12294,6 +12510,44 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn list_path_producer_worker_preserves_s3_and_caller_context() {
|
||||
let mut request = rustfs_io_metrics::s3_http_metrics::S3HttpRequestGuard::new("GET");
|
||||
let api_operation = request
|
||||
.in_scope(|| {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::ListObjectsV2);
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_context()
|
||||
})
|
||||
.expect("dispatched list operation should be captured");
|
||||
request.response(200);
|
||||
|
||||
let (sender, recv) = mpsc::channel(1);
|
||||
let opts = ListPathOptions {
|
||||
caller: rustfs_io_metrics::ListObjectsGatherCaller::S3List,
|
||||
..Default::default()
|
||||
};
|
||||
let result = rustfs_io_metrics::s3_http_metrics::with_s3_http_operation_context(Some(api_operation), async move {
|
||||
collect_list_path_results(CancellationToken::new(), opts, recv, async move {
|
||||
assert_eq!(
|
||||
rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(),
|
||||
Some("s3:ListObjectsV2")
|
||||
);
|
||||
assert_eq!(
|
||||
crate::cluster::rpc::runtime_sources::current_read_version_request_caller(),
|
||||
Some(rustfs_io_metrics::internode_metrics::ReadVersionRequestCaller::ListObjects)
|
||||
);
|
||||
drop(sender);
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(result.err.is_some(), "the empty page channel remains an error");
|
||||
assert_eq!(rustfs_io_metrics::s3_http_metrics::current_s3_http_operation_name(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_workers_preserve_error_after_input_closes() {
|
||||
let (sender, recv) = mpsc::channel(1);
|
||||
|
||||
@@ -0,0 +1,498 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::lifecycle::core::IlmAction;
|
||||
use crate::bucket::lifecycle::evaluator::Evaluator;
|
||||
use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, BUCKET_VERSIONING_CONFIG};
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::disk::lifecycle_test_io::Observer;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::object_api::PutObjReader;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use crate::storage_api_contracts::multipart::{CompletePart, MultipartOperations};
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo};
|
||||
use tempfile::TempDir;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
async fn fixture(pool_count: usize) -> (Vec<TempDir>, Arc<ECStore>, String, Uuid) {
|
||||
let mut dirs = Vec::new();
|
||||
let mut pools = Vec::new();
|
||||
for pool_idx in 0..pool_count {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_idx in 0..4 {
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let mut endpoint =
|
||||
crate::disk::endpoint::Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("disk endpoint");
|
||||
endpoint.set_pool_index(pool_idx);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_idx);
|
||||
dirs.push(dir);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("lifecycle-exact-pool-{pool_idx}"),
|
||||
platform: "test".into(),
|
||||
});
|
||||
}
|
||||
let endpoints = EndpointServerPools(pools);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
|
||||
.await
|
||||
.expect("initialize disks");
|
||||
let store =
|
||||
ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("test address"), endpoints, CancellationToken::new(), ctx)
|
||||
.await
|
||||
.expect("initialize store");
|
||||
metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("lifecycle-exact-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = metadata_sys::get_bucket_incarnation_id_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
(dirs, store, bucket, incarnation)
|
||||
}
|
||||
|
||||
fn version(object: &str, id: Option<Uuid>, timestamp: i64) -> FileInfo {
|
||||
let mut fi = FileInfo::new(object, 2, 2);
|
||||
fi.name = object.to_owned();
|
||||
fi.version_id = id;
|
||||
fi.versioned = id.is_some();
|
||||
fi.data_dir = Some(Uuid::from_u128(0x1234));
|
||||
fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(timestamp).expect("version timestamp"));
|
||||
fi.size = 1;
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1,
|
||||
actual_size: 1,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.metadata.insert("etag".into(), timestamp.to_string());
|
||||
fi
|
||||
}
|
||||
|
||||
async fn seed(dirs: &[TempDir], bucket: &str, object: &str, versions: &[FileInfo]) {
|
||||
for (index, dir) in dirs.iter().enumerate() {
|
||||
let mut meta = FileMeta::new();
|
||||
for version in versions {
|
||||
let mut fi = version.clone();
|
||||
fi.volume = bucket.to_owned();
|
||||
if !fi.is_canonical_delete_marker() {
|
||||
fi.erasure.index = index % 4 + 1;
|
||||
}
|
||||
fi.validate_for_metadata_read().expect("valid fixture version");
|
||||
meta.add_version(fi).expect("encode version");
|
||||
}
|
||||
let path = dir.path().join(bucket).join(path::encode_dir_object(object));
|
||||
tokio::fs::create_dir_all(&path).await.expect("object directory");
|
||||
tokio::fs::write(path.join(crate::disk::STORAGE_FORMAT_FILE), meta.marshal_msg().expect("encode metadata"))
|
||||
.await
|
||||
.expect("persist fixture metadata");
|
||||
}
|
||||
}
|
||||
|
||||
async fn configure_versioning(store: &ECStore, bucket: &str, state: &str) {
|
||||
if state == "Unversioned" {
|
||||
return;
|
||||
}
|
||||
metadata_sys::update_in(
|
||||
&store.ctx,
|
||||
bucket,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
format!("<VersioningConfiguration><Status>{state}</Status></VersioningConfiguration>").into_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("persist versioning configuration");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_exact_matches_listing_for_complete_version_groups() {
|
||||
let (dirs, store, bucket, incarnation) = fixture(1).await;
|
||||
for state in ["Unversioned", "Enabled", "Suspended"] {
|
||||
configure_versioning(&store, &bucket, state).await;
|
||||
for object in ["abc", "nested/abc", "directory/"] {
|
||||
let mut versions = vec![version(object, None, 1_700_000_000)];
|
||||
if state != "Unversioned" {
|
||||
versions.push(version(object, Some(Uuid::from_u128(1)), 1_700_000_001));
|
||||
versions.push(FileInfo {
|
||||
name: object.into(),
|
||||
version_id: Some(Uuid::from_u128(2)),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(1_700_000_002).expect("marker timestamp")),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
seed(&dirs, &bucket, object, &versions).await;
|
||||
let sibling = format!("{object}-sibling");
|
||||
seed(&dirs, &bucket, &sibling, &[version(&sibling, None, 1_700_000_003)]).await;
|
||||
let observed = Observer::new(&bucket);
|
||||
let exact = store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &ObjectOptions::default(), incarnation)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("read {state} versions for {object}: {error}"));
|
||||
let counts = observed.snapshot();
|
||||
assert_eq!(counts.walks, 0, "exact metadata reads must never enumerate a directory");
|
||||
assert_eq!(counts.reads.len(), 4, "one exact metadata read per physical drive");
|
||||
assert!(counts.reads.iter().all(|key| key == &path::encode_dir_object(object)));
|
||||
let mut listed = store
|
||||
.clone()
|
||||
.list_object_versions_for_lifecycle(&bucket, object, None, None, None, 1000)
|
||||
.await
|
||||
.expect("reference lifecycle listing")
|
||||
.objects
|
||||
.into_iter()
|
||||
.filter(|info| info.name == object)
|
||||
.collect::<Vec<_>>();
|
||||
// The legacy list projection uses ambient bucket versioning. This
|
||||
// fixture has an isolated store; restore its known null-version
|
||||
// representation before comparing the version-group semantics.
|
||||
if state != "Unversioned" {
|
||||
for info in &mut listed {
|
||||
if info.version_id.is_none() {
|
||||
info.version_id = Some(Uuid::nil());
|
||||
}
|
||||
}
|
||||
assert!(exact.iter().any(|info| info.version_id == Some(Uuid::nil())));
|
||||
}
|
||||
assert!(observed.snapshot().walks > 0, "the old prefix-listing path must exercise the observer");
|
||||
assert_eq!(exact.len(), versions.len(), "{state}: {object}");
|
||||
assert_eq!(exact.len(), listed.len());
|
||||
for (exact, listed) in exact.iter().zip(&listed) {
|
||||
assert_eq!(
|
||||
(
|
||||
exact.version_id,
|
||||
exact.is_latest,
|
||||
exact.delete_marker,
|
||||
exact.num_versions,
|
||||
exact.successor_mod_time
|
||||
),
|
||||
(
|
||||
listed.version_id,
|
||||
listed.is_latest,
|
||||
listed.delete_marker,
|
||||
listed.num_versions,
|
||||
listed.successor_mod_time
|
||||
)
|
||||
);
|
||||
assert_eq!(exact.user_defined, listed.user_defined);
|
||||
assert_eq!(exact.version_purge_status, listed.version_purge_status);
|
||||
}
|
||||
let policy = crate::bucket::utils::deserialize(
|
||||
br#"<LifecycleConfiguration><Rule><ID>history</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Days>30</Days></Expiration><NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays><NewerNoncurrentVersions>1</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule></LifecycleConfiguration>"#,
|
||||
).expect("reference policy");
|
||||
let evaluator = Evaluator::new(Arc::new(policy));
|
||||
let to_opts = |objects: &[ObjectInfo]| {
|
||||
objects
|
||||
.iter()
|
||||
.map(crate::bucket::lifecycle::core::object_opts_from_object_info)
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let exact_events = evaluator.eval(&to_opts(&exact)).await.expect("evaluate exact group");
|
||||
let listed_events = evaluator.eval(&to_opts(&listed)).await.expect("evaluate listed group");
|
||||
for (exact, listed) in exact_events.iter().zip(listed_events) {
|
||||
assert_eq!((exact.action, exact.due, &exact.rule_id), (listed.action, listed.due, &listed.rule_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_exact_merges_pools_and_more_than_one_page_of_versions() {
|
||||
let (dirs, store, bucket, incarnation) = fixture(2).await;
|
||||
configure_versioning(&store, &bucket, "Enabled").await;
|
||||
let object = "history";
|
||||
let versions = (0..1002)
|
||||
.map(|index| {
|
||||
version(
|
||||
object,
|
||||
Some(Uuid::from_u128(index + 1)),
|
||||
1_700_000_000 + i64::try_from(index).expect("timestamp index"),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
seed(&dirs[..4], &bucket, object, &versions[..601]).await;
|
||||
seed(&dirs[4..], &bucket, object, &versions[600..]).await;
|
||||
let observed = Observer::new(&bucket);
|
||||
let exact = store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &ObjectOptions::default(), incarnation)
|
||||
.await
|
||||
.expect("merge independently authoritative pools");
|
||||
assert_eq!(exact.len(), 1002, "overlapping migration copies must be deduplicated without pagination");
|
||||
assert_eq!(exact[0].version_id, Some(Uuid::from_u128(1002)));
|
||||
assert!(exact[0].is_latest);
|
||||
assert!(exact.iter().all(|info| info.num_versions == 1002));
|
||||
for pair in exact.windows(2) {
|
||||
assert!(!pair[1].is_latest);
|
||||
assert_eq!(pair[1].successor_mod_time, pair[0].mod_time);
|
||||
}
|
||||
assert_eq!(observed.snapshot().walks, 0);
|
||||
assert_eq!(observed.snapshot().reads.len(), 8, "read count depends on drives, not version pages");
|
||||
let mut conflict = versions[600].clone();
|
||||
conflict.metadata.insert("etag".into(), "conflicting-content".into());
|
||||
seed(&dirs[4..], &bucket, object, &[conflict]).await;
|
||||
assert!(
|
||||
store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &ObjectOptions::default(), incarnation)
|
||||
.await
|
||||
.is_err(),
|
||||
"conflicting cross-pool copies cannot authorize expiry"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_exact_rejects_uncertain_metadata_and_stale_bucket_identity() {
|
||||
let (dirs, store, bucket, incarnation) = fixture(1).await;
|
||||
let object = "probe";
|
||||
let opts = ObjectOptions::default();
|
||||
assert!(
|
||||
store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &opts, incarnation)
|
||||
.await
|
||||
.expect("confirmed absence")
|
||||
.is_empty()
|
||||
);
|
||||
seed(&dirs, &bucket, object, &[version(object, None, 1_700_000_000)]).await;
|
||||
assert!(
|
||||
store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &opts, Uuid::new_v4())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for dir in &dirs {
|
||||
tokio::fs::write(
|
||||
dir.path()
|
||||
.join(bucket.as_str())
|
||||
.join(object)
|
||||
.join(crate::disk::STORAGE_FORMAT_FILE),
|
||||
b"corrupt",
|
||||
)
|
||||
.await
|
||||
.expect("corrupt fixture metadata");
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &opts, incarnation)
|
||||
.await
|
||||
.is_err(),
|
||||
"corruption is not absence"
|
||||
);
|
||||
seed(&dirs, &bucket, object, &[version(object, None, 1_700_000_000)]).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let saved = set.disks.read().await.clone();
|
||||
*set.disks.write().await = vec![saved[0].clone(), saved[1].clone(), None, None];
|
||||
assert_eq!(
|
||||
store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &opts, incarnation)
|
||||
.await
|
||||
.expect("exactly the object's read quorum")
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
*set.disks.write().await = vec![saved[0].clone(), None, None, None];
|
||||
assert!(
|
||||
store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &opts, incarnation)
|
||||
.await
|
||||
.is_err(),
|
||||
"a minority cannot authorize expiry"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.get_object_versions_for_lifecycle(&bucket, "missing", &opts, incarnation)
|
||||
.await
|
||||
.is_err(),
|
||||
"missing on one drive does not prove absence"
|
||||
);
|
||||
*set.disks.write().await = saved;
|
||||
let lost = ObjectOptions {
|
||||
no_lock: true,
|
||||
namespace_lock_fence: Some(crate::object_api::NamespaceLockFence::lost_for_test()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(
|
||||
store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &lost, incarnation)
|
||||
.await
|
||||
.is_err(),
|
||||
"an inherited lost object fence must reject the snapshot"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_exact_write_hooks_never_walk_parent_directories() {
|
||||
let (dirs, store, bucket, _) = fixture(1).await;
|
||||
metadata_sys::update_in(&store.ctx, &bucket, BUCKET_LIFECYCLE_CONFIG,
|
||||
br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#.to_vec())
|
||||
.await.expect("configure expiration");
|
||||
for index in 0..64 {
|
||||
let key = format!("probe-{index}");
|
||||
seed(&dirs, &bucket, &key, &[version(&key, None, 1_700_000_000)]).await;
|
||||
}
|
||||
for state in ["Unversioned", "Enabled", "Suspended"] {
|
||||
configure_versioning(&store, &bucket, state).await;
|
||||
let opts = ObjectOptions {
|
||||
versioned: state == "Enabled",
|
||||
version_suspended: state == "Suspended",
|
||||
..Default::default()
|
||||
};
|
||||
let observer = Observer::new(&bucket);
|
||||
let mut source = store
|
||||
.put_object(&bucket, "probe", &mut PutObjReader::from_vec(b"payload".to_vec()), &opts)
|
||||
.await
|
||||
.expect("PUT with lifecycle");
|
||||
source.put_object_reader = Some(PutObjReader::from_vec(b"payload".to_vec()));
|
||||
let copied = store
|
||||
.copy_object(&bucket, "probe", &bucket, "copy", &mut source, &ObjectOptions::default(), &opts)
|
||||
.await
|
||||
.expect("COPY with lifecycle");
|
||||
assert_eq!(copied.size, source.size);
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, "multipart", &opts)
|
||||
.await
|
||||
.expect("init multipart");
|
||||
let part = store
|
||||
.put_object_part(
|
||||
&bucket,
|
||||
"multipart",
|
||||
&upload.upload_id,
|
||||
1,
|
||||
&mut PutObjReader::from_vec(b"part".to_vec()),
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("upload part");
|
||||
let complete = store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
"multipart",
|
||||
&upload.upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: part.etag,
|
||||
checksum_crc32: None,
|
||||
checksum_crc32c: None,
|
||||
checksum_sha1: None,
|
||||
checksum_sha256: None,
|
||||
checksum_crc64nvme: None,
|
||||
}],
|
||||
&opts,
|
||||
)
|
||||
.await
|
||||
.expect("complete multipart with lifecycle");
|
||||
assert_eq!(complete.size, 4);
|
||||
let counts = observer.snapshot();
|
||||
assert_eq!(counts.walks, 0, "{state}: successful writes must not run the version walker");
|
||||
for key in ["probe", "copy", "multipart"] {
|
||||
assert!(
|
||||
counts.reads.iter().filter(|read| read.as_str() == key).count() >= 4,
|
||||
"{state}: {key} must run exact expiry evaluation"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
!counts.reads.iter().any(|key| key.starts_with("probe-")),
|
||||
"prefix siblings must never enter evaluation"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_exact_keeps_pending_purges_visible_to_expiry_guards() {
|
||||
let (dirs, store, bucket, incarnation) = fixture(1).await;
|
||||
configure_versioning(&store, &bucket, "Enabled").await;
|
||||
let object = "pending";
|
||||
let (entry, _) = super::test::test_pending_version_purge_meta_entry(&bucket, object);
|
||||
for dir in &dirs {
|
||||
let path = dir.path().join(&bucket).join(object);
|
||||
tokio::fs::create_dir_all(&path).await.expect("object directory");
|
||||
tokio::fs::write(path.join(crate::disk::STORAGE_FORMAT_FILE), &entry.metadata)
|
||||
.await
|
||||
.expect("persist pending purge");
|
||||
}
|
||||
let exact = store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &ObjectOptions::default(), incarnation)
|
||||
.await
|
||||
.expect("read pending purge");
|
||||
assert_eq!(exact.len(), 1);
|
||||
assert_eq!(exact[0].num_versions, 1);
|
||||
assert_eq!(exact[0].version_purge_status, crate::bucket::replication::VersionPurgeStatusType::Pending);
|
||||
let policy = crate::bucket::utils::deserialize(
|
||||
br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>"#,
|
||||
).expect("expiration policy");
|
||||
let opts = crate::bucket::lifecycle::core::object_opts_from_object_info(&exact[0]);
|
||||
let events = Evaluator::new(Arc::new(policy))
|
||||
.eval(&[opts])
|
||||
.await
|
||||
.expect("evaluate pending purge");
|
||||
assert_eq!(events[0].action, IlmAction::NoneAction);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_exact_io_is_independent_of_parent_directory_width() {
|
||||
let (dirs, store, bucket, incarnation) = fixture(1).await;
|
||||
let object = "probe";
|
||||
seed(&dirs, &bucket, object, &[version(object, None, 1_700_000_000)]).await;
|
||||
let mut previous = 0;
|
||||
for width in [64, 1024, 8192] {
|
||||
let roots = dirs.iter().map(|dir| dir.path().join(&bucket)).collect::<Vec<_>>();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
for root in roots {
|
||||
for index in previous..width {
|
||||
std::fs::create_dir(root.join(format!("sibling-{index:05}"))).expect("parent-directory entry");
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("create wide directory");
|
||||
previous = width;
|
||||
let mut exact_times = Vec::new();
|
||||
let mut listing_times = Vec::new();
|
||||
for _ in 0..5 {
|
||||
let observer = Observer::new(&bucket);
|
||||
let started = std::time::Instant::now();
|
||||
let exact = store
|
||||
.get_object_versions_for_lifecycle(&bucket, object, &ObjectOptions::default(), incarnation)
|
||||
.await
|
||||
.expect("exact lifecycle read");
|
||||
exact_times.push(started.elapsed().as_micros());
|
||||
assert_eq!(exact.len(), 1);
|
||||
assert_eq!(observer.snapshot().walks, 0);
|
||||
assert_eq!(observer.snapshot().reads.len(), 4, "directory width {width}");
|
||||
let started = std::time::Instant::now();
|
||||
let listed = store
|
||||
.clone()
|
||||
.list_object_versions_for_lifecycle(&bucket, object, None, None, None, 1000)
|
||||
.await
|
||||
.expect("reference prefix scan");
|
||||
listing_times.push(started.elapsed().as_micros());
|
||||
assert_eq!(listed.objects.len(), 1);
|
||||
assert!(observer.snapshot().walks > 0);
|
||||
}
|
||||
exact_times.sort_unstable();
|
||||
listing_times.sort_unstable();
|
||||
eprintln!(
|
||||
"lifecycle directory width={width}: exact median={}us max={}us; prefix median={}us max={}us",
|
||||
exact_times[2], exact_times[4], listing_times[2], listing_times[4]
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -428,6 +428,7 @@ mod bucket;
|
||||
mod bucket_fence;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
pub use bucket_fence::BucketIncarnationFenceGuard;
|
||||
pub(crate) mod delete_recovery_runtime;
|
||||
mod heal;
|
||||
pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
pub(crate) use heal::{bucket_heal_scope, bucket_heal_scope_for_object};
|
||||
@@ -1363,7 +1364,13 @@ impl crate::storage_api_contracts::object::ObjectIO for ECStore {
|
||||
h: HeaderMap,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<GetObjectReader> {
|
||||
self.handle_get_object_reader(bucket, object, range, h, opts).await
|
||||
let read = self.handle_get_object_reader(bucket, object, range, h, opts);
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
let read = crate::cluster::rpc::runtime_sources::with_read_version_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::StoreObjectReader,
|
||||
read,
|
||||
);
|
||||
read.await
|
||||
}
|
||||
#[instrument(level = "debug", skip(self, data))]
|
||||
async fn put_object(&self, bucket: &str, object: &str, data: &mut PutObjReader, opts: &ObjectOptions) -> Result<ObjectInfo> {
|
||||
|
||||
@@ -32,6 +32,42 @@ fn data_movement_multipart_discovery_counts() -> &'static std::sync::Mutex<std::
|
||||
DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type OrphanMultipartCleanupPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
#[cfg(test)]
|
||||
static ORPHAN_MULTIPART_CLEANUP_PAUSES: std::sync::OnceLock<
|
||||
std::sync::Mutex<std::collections::HashMap<String, OrphanMultipartCleanupPause>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn pause_orphan_multipart_cleanup_for_test(
|
||||
bucket: &str,
|
||||
) -> (tokio::sync::oneshot::Receiver<()>, tokio::sync::oneshot::Sender<()>) {
|
||||
let (arrived_tx, arrived_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let previous = ORPHAN_MULTIPART_CLEANUP_PAUSES
|
||||
.get_or_init(Default::default)
|
||||
.lock()
|
||||
.expect("orphan cleanup pause mutex should not poison")
|
||||
.insert(bucket.to_owned(), (arrived_tx, release_rx));
|
||||
assert!(previous.is_none(), "each bucket has at most one orphan cleanup pause");
|
||||
(arrived_rx, release_tx)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_orphan_multipart_cleanup(bucket: &str) {
|
||||
let pause = ORPHAN_MULTIPART_CLEANUP_PAUSES
|
||||
.get_or_init(Default::default)
|
||||
.lock()
|
||||
.expect("orphan cleanup pause mutex should not poison")
|
||||
.remove(bucket);
|
||||
if let Some((arrived, release)) = pause {
|
||||
let _ = arrived.send(());
|
||||
let _ = release.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn decommission_multipart_target_clear_pending(opts: &ObjectOptions, target: Option<&ObjectInfo>) -> Result<bool> {
|
||||
let expected_mod_time = opts.mod_time.ok_or_else(|| Error::DecommissionCapacityBlocked {
|
||||
message: "multipart cleanup cannot prove exact target absence without a modification time".to_string(),
|
||||
@@ -654,7 +690,10 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if let Some(guard) = bucket_lifecycle_guard {
|
||||
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
|
||||
}
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -750,19 +789,12 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
|
||||
// TODO(backlog): defer DeleteUploadID to background for faster abort response
|
||||
|
||||
if self.single_pool() {
|
||||
return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;
|
||||
}
|
||||
let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
let pool = &self.pools[pool_idx];
|
||||
|
||||
let err = match pool.abort_multipart_upload(bucket, object, upload_id, opts).await {
|
||||
let err = match pool.abort_multipart_upload(bucket, object, upload_id, &opts).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(err) => {
|
||||
//
|
||||
@@ -775,7 +807,92 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||
self.recover_orphan_multipart_upload(bucket, object, upload_id, opts, bucket_lifecycle_guard)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn recover_orphan_multipart_upload(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
opts: ObjectOptions,
|
||||
bucket_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
let Some(bucket_lifecycle_guard) = bucket_lifecycle_guard else {
|
||||
// A signal borrowed from an enclosing operation cannot keep its lock
|
||||
// alive when the request waiter disappears during physical cleanup.
|
||||
return Err(invalid_upload());
|
||||
};
|
||||
let Some(expected_incarnation) = opts.expected_bucket_incarnation_id.filter(|id| !id.is_nil()) else {
|
||||
return Err(invalid_upload());
|
||||
};
|
||||
// Recovery is a side path of an idempotent Abort. Until the first
|
||||
// deletion is issued, anything that stops staging absence from being
|
||||
// proven (an offline disk, lock contention, partially overlapping lock
|
||||
// domains, unreadable bucket metadata) keeps the upload reported as
|
||||
// gone, exactly as before recovery existed; only failures inside the
|
||||
// deletion task are request failures, because a retry can finish them.
|
||||
let (metadata, persisted) =
|
||||
match crate::bucket::metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await {
|
||||
Ok(loaded) => loaded,
|
||||
Err(error) => {
|
||||
debug!(bucket, object, upload_id, %error, "orphan multipart cleanup skipped: bucket metadata unavailable");
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
};
|
||||
if !persisted
|
||||
|| !metadata.bucket_creation_committed
|
||||
|| !metadata.bucket_incarnation_sidecar
|
||||
|| metadata.bucket_incarnation_id != expected_incarnation
|
||||
{
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
|
||||
// Lock order: bucket lifecycle, then upload locks in pool-index order.
|
||||
// Prove metadata absence across every pool before deleting any staging.
|
||||
let mut cleanup_guards = Vec::with_capacity(self.pools.len());
|
||||
for pool in &self.pools {
|
||||
let guard = match pool
|
||||
.get_disks_by_key(object)
|
||||
.lock_orphan_multipart_cleanup(bucket, object, upload_id, &opts, &cleanup_guards)
|
||||
.await
|
||||
{
|
||||
Ok(guard) => guard,
|
||||
Err(error) => {
|
||||
if !is_err_invalid_upload_id(&error) {
|
||||
debug!(bucket, object, upload_id, %error, "orphan multipart cleanup skipped: staging absence not proven");
|
||||
}
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
};
|
||||
cleanup_guards.push(guard);
|
||||
}
|
||||
let bucket = bucket.to_owned();
|
||||
let object = object.to_owned();
|
||||
let upload_id = upload_id.to_owned();
|
||||
tokio::spawn(async move {
|
||||
// Own both the real bucket guard and all upload guards until every
|
||||
// disk deletion settles, including after cancellation of the waiter.
|
||||
let _bucket_lifecycle_guard = bucket_lifecycle_guard;
|
||||
#[cfg(test)]
|
||||
pause_orphan_multipart_cleanup(&bucket).await;
|
||||
let mut recovered = false;
|
||||
for guard in &cleanup_guards {
|
||||
recovered |= guard.delete(&bucket, &object, &opts).await?;
|
||||
}
|
||||
if recovered {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StorageError::InvalidUploadID(bucket, object, upload_id))
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::other_with_context("orphan multipart cleanup task failed", error))?
|
||||
}
|
||||
|
||||
pub(crate) async fn abort_multipart_upload_for_data_movement(
|
||||
|
||||
+1328
-103
File diff suppressed because it is too large
Load Diff
@@ -85,6 +85,13 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Resolve a quota lease against this request's captured storage instance.
|
||||
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
Ok(disk)
|
||||
}
|
||||
|
||||
pub async fn write_local_metadata_at_incarnation(
|
||||
self: &Arc<Self>,
|
||||
disk_ref: &str,
|
||||
|
||||
@@ -125,9 +125,8 @@ async fn pause_prepared_pool_read_before_refetch(object: &str) {
|
||||
#[cfg(test)]
|
||||
use support::resolve_latest_object_info_candidates;
|
||||
use support::{
|
||||
LatestObjectInfoCandidate, PoolErr, PoolObjInfo, RebalanceDeletePoolResult, pool_lookup_not_found_error,
|
||||
rebalance_disk_set_lookup_error, resolve_latest_object_info_candidates_with_pool_state,
|
||||
resolve_rebalance_delete_from_all_pools_result, resolve_rebalance_delete_from_all_pools_results,
|
||||
LatestObjectInfoCandidate, PoolErr, PoolObjInfo, pool_lookup_not_found_error, rebalance_disk_set_lookup_error,
|
||||
resolve_latest_object_info_candidates_with_pool_state, resolve_rebalance_delete_from_all_pools_results,
|
||||
resolve_store_rebalance_pool_meta_reload_result, validate_prepared_pool_refetch_identity,
|
||||
};
|
||||
|
||||
@@ -996,42 +995,27 @@ impl ECStore {
|
||||
|
||||
let mut results = Vec::with_capacity(errs.len());
|
||||
|
||||
for pe in errs.iter() {
|
||||
if let Some(err) = &pe.err
|
||||
&& err == &StorageError::ErasureWriteQuorum
|
||||
{
|
||||
if let Some(idx) = pe.index {
|
||||
results.push(RebalanceDeletePoolResult {
|
||||
pool_idx: idx,
|
||||
result: Err(StorageError::ErasureWriteQuorum),
|
||||
});
|
||||
}
|
||||
for pe in &errs {
|
||||
let Some(idx) = pe.index else { continue };
|
||||
if pe.err.as_ref() == Some(&StorageError::ErasureWriteQuorum) {
|
||||
results.push(Err(StorageError::ErasureWriteQuorum));
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(idx) = pe.index {
|
||||
let pool = self.pools[idx].clone();
|
||||
results.push(RebalanceDeletePoolResult {
|
||||
pool_idx: idx,
|
||||
result: self
|
||||
.run_external_decommission_capacity_object_delete(
|
||||
idx,
|
||||
bucket,
|
||||
object,
|
||||
object,
|
||||
opts.clone(),
|
||||
|opts| async move { pool.delete_object(bucket, object, opts).await },
|
||||
)
|
||||
.await,
|
||||
});
|
||||
}
|
||||
let pool = self.pools[idx].clone();
|
||||
results.push(
|
||||
self.run_external_decommission_capacity_object_delete(
|
||||
idx,
|
||||
bucket,
|
||||
object,
|
||||
object,
|
||||
opts.clone(),
|
||||
|opts| async move { pool.delete_object(bucket, object, opts).await },
|
||||
)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
|
||||
resolve_rebalance_delete_from_all_pools_result(
|
||||
resolve_rebalance_delete_from_all_pools_results(results, bucket, object),
|
||||
bucket,
|
||||
object,
|
||||
)
|
||||
resolve_rebalance_delete_from_all_pools_results(results, bucket, object)
|
||||
}
|
||||
|
||||
/// Peer reload entry: refreshes in-memory pool metadata from the shared
|
||||
@@ -2460,56 +2444,22 @@ mod tests {
|
||||
assert!(err_message.contains(&Error::SlowDown.to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_rebalance_delete_from_all_pools_result_passthrough_ok() {
|
||||
let info = ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let resolved = resolve_rebalance_delete_from_all_pools_result(Ok(info.clone()), "bucket", "object")
|
||||
.expect("successful rebalance delete should pass through");
|
||||
|
||||
assert_eq!(resolved.bucket, info.bucket);
|
||||
assert_eq!(resolved.name, info.name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_rebalance_delete_from_all_pools_result_wraps_object_context() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_result(Err(Error::SlowDown), "bucket", "object")
|
||||
.expect_err("failed rebalance delete should be wrapped");
|
||||
let rendered = err.to_string();
|
||||
|
||||
assert!(rendered.contains("failed to delete rebalance source object bucket/object"), "{rendered}");
|
||||
assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_rebalance_delete_from_all_pools_results_fails_on_later_pool_error() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_results(
|
||||
vec![
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 0,
|
||||
result: Ok(ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
},
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 1,
|
||||
result: Err(Error::SlowDown),
|
||||
},
|
||||
Ok(ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
Err(Error::SlowDown),
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("non-ignorable errors from later pools must not be hidden");
|
||||
let rendered = err.to_string();
|
||||
|
||||
assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
|
||||
assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
|
||||
assert_eq!(err, Error::SlowDown);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2522,14 +2472,8 @@ mod tests {
|
||||
|
||||
let resolved = resolve_rebalance_delete_from_all_pools_results(
|
||||
vec![
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 0,
|
||||
result: Ok(info.clone()),
|
||||
},
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 1,
|
||||
result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
|
||||
},
|
||||
Ok(info.clone()),
|
||||
Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
@@ -2550,14 +2494,8 @@ mod tests {
|
||||
|
||||
let resolved = resolve_rebalance_delete_from_all_pools_results(
|
||||
vec![
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 0,
|
||||
result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
|
||||
},
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 1,
|
||||
result: Ok(info.clone()),
|
||||
},
|
||||
Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
|
||||
Ok(info.clone()),
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
@@ -2572,50 +2510,35 @@ mod tests {
|
||||
fn resolve_rebalance_delete_from_all_pools_results_fails_when_all_results_are_ignored_errors() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_results(
|
||||
vec![
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 0,
|
||||
result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
|
||||
},
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 1,
|
||||
result: Err(Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())),
|
||||
},
|
||||
Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
|
||||
Err(Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())),
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("all ignored errors without any successful delete should still fail");
|
||||
let rendered = err.to_string();
|
||||
|
||||
assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
|
||||
assert!(rendered.contains("Version not found"), "{rendered}");
|
||||
assert_eq!(
|
||||
err,
|
||||
Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_rebalance_delete_from_all_pools_results_fails_on_write_quorum_even_with_success() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_results(
|
||||
vec![
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 0,
|
||||
result: Ok(ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
},
|
||||
RebalanceDeletePoolResult {
|
||||
pool_idx: 1,
|
||||
result: Err(Error::ErasureWriteQuorum),
|
||||
},
|
||||
Ok(ObjectInfo {
|
||||
bucket: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
Err(Error::ErasureWriteQuorum),
|
||||
],
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("write quorum failures must fail the aggregate delete");
|
||||
let rendered = err.to_string();
|
||||
|
||||
assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
|
||||
assert!(rendered.contains(&Error::ErasureWriteQuorum.to_string()), "{rendered}");
|
||||
assert_eq!(err, Error::ErasureWriteQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -46,15 +46,10 @@ impl Clone for PoolObjInfo {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct LatestObjectInfoCandidate {
|
||||
pub(super) info: Option<ObjectInfo>,
|
||||
pub(super) idx: usize,
|
||||
pub(super) err: Option<Error>,
|
||||
}
|
||||
|
||||
pub(super) struct RebalanceDeletePoolResult {
|
||||
pub(super) pool_idx: usize,
|
||||
pub(super) result: Result<ObjectInfo>,
|
||||
pub(in crate::store) struct LatestObjectInfoCandidate {
|
||||
pub(in crate::store) info: Option<ObjectInfo>,
|
||||
pub(in crate::store) idx: usize,
|
||||
pub(in crate::store) err: Option<Error>,
|
||||
}
|
||||
|
||||
pub(super) fn pool_lookup_not_found_error(bucket: &str, object: &str, opts: &ObjectOptions) -> Error {
|
||||
@@ -71,53 +66,30 @@ pub(super) fn resolve_store_rebalance_pool_meta_reload_result<T>(result: Result<
|
||||
result.map_err(|err| Error::other(format!("store rebalance pool meta reload failed during {stage}: {err}")))
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_delete_from_all_pools_result(
|
||||
result: Result<ObjectInfo>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<ObjectInfo> {
|
||||
result.map_err(|err| {
|
||||
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
|
||||
err
|
||||
} else {
|
||||
Error::other(format!("failed to delete rebalance source object {bucket}/{object}: {err}"))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn is_ignorable_rebalance_delete_error(err: &Error) -> bool {
|
||||
is_err_object_not_found(err) || is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
fn rebalance_delete_pool_error(pool_idx: usize, bucket: &str, object: &str, err: Error) -> Error {
|
||||
if matches!(&err, Error::PreconditionFailed | Error::PrefixAccessDenied(_, _)) {
|
||||
err
|
||||
} else {
|
||||
Error::other(format!("pool {pool_idx} delete failed for {bucket}/{object}: {err}"))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn resolve_rebalance_delete_from_all_pools_results(
|
||||
results: Vec<RebalanceDeletePoolResult>,
|
||||
results: Vec<Result<ObjectInfo>>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<ObjectInfo> {
|
||||
let mut deleted = None;
|
||||
let mut ignored_error = None;
|
||||
|
||||
for pool_result in results {
|
||||
let pool_idx = pool_result.pool_idx;
|
||||
match pool_result.result {
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(info) => {
|
||||
if deleted.is_none() {
|
||||
deleted = Some(info);
|
||||
}
|
||||
}
|
||||
Err(err) if is_ignorable_rebalance_delete_error(&err) => {
|
||||
ignored_error = Some((pool_idx, err));
|
||||
ignored_error = Some(err);
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -126,8 +98,8 @@ pub(super) fn resolve_rebalance_delete_from_all_pools_results(
|
||||
return Ok(info);
|
||||
}
|
||||
|
||||
if let Some((pool_idx, err)) = ignored_error {
|
||||
return Err(rebalance_delete_pool_error(pool_idx, bucket, object, err));
|
||||
if let Some(err) = ignored_error {
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Err(Error::other(format!(
|
||||
@@ -295,7 +267,7 @@ pub(super) fn resolve_latest_object_info_candidates(
|
||||
resolve_latest_object_info_candidates_with_pool_state(candidates, &[], bucket, object, opts)
|
||||
}
|
||||
|
||||
pub(super) fn resolve_latest_object_info_candidates_with_pool_state(
|
||||
pub(in crate::store) fn resolve_latest_object_info_candidates_with_pool_state(
|
||||
candidates: Vec<LatestObjectInfoCandidate>,
|
||||
suspended_pools: &[bool],
|
||||
bucket: &str,
|
||||
@@ -377,52 +349,56 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_result_preserves_precondition_failed() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_result(Err(Error::PreconditionFailed), "bucket", "object")
|
||||
.expect_err("precondition failure should remain structured");
|
||||
|
||||
assert_eq!(err, Error::PreconditionFailed);
|
||||
fn rebalance_delete_aggregation_preserves_typed_failures() {
|
||||
for expected in [
|
||||
Error::UnsupportedDisk,
|
||||
Error::GuardedDeleteUnsupported,
|
||||
Error::ErasureReadQuorum,
|
||||
Error::ErasureWriteQuorum,
|
||||
Error::SlowDown,
|
||||
Error::PreconditionFailed,
|
||||
Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()),
|
||||
Error::FileCorrupt,
|
||||
] {
|
||||
for results in [
|
||||
vec![Err(expected.clone())],
|
||||
vec![Ok(ObjectInfo::default()), Err(expected.clone())],
|
||||
vec![Err(expected.clone()), Ok(ObjectInfo::default())],
|
||||
vec![
|
||||
Err(Error::ObjectNotFound("bucket".to_owned(), "object".to_owned())),
|
||||
Err(expected.clone()),
|
||||
],
|
||||
] {
|
||||
let actual = resolve_rebalance_delete_from_all_pools_results(results, "bucket", "object")
|
||||
.expect_err("a pool failure must retain its S3 error classification");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_result_preserves_prefix_access_denied() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_result(
|
||||
Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("prefix access denial should remain structured");
|
||||
|
||||
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
|
||||
fn rebalance_delete_aggregation_preserves_missing_identity() {
|
||||
for expected in [
|
||||
Error::ObjectNotFound("bucket".to_owned(), "object".to_owned()),
|
||||
Error::VersionNotFound("bucket".to_owned(), "object".to_owned(), "version".to_owned()),
|
||||
] {
|
||||
let actual = resolve_rebalance_delete_from_all_pools_results(vec![Err(expected.clone())], "bucket", "object")
|
||||
.expect_err("all-missing deletes must preserve their typed object or version identity");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_pool_result_preserves_precondition_failed() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_results(
|
||||
vec![RebalanceDeletePoolResult {
|
||||
pool_idx: 0,
|
||||
result: Err(Error::PreconditionFailed),
|
||||
}],
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("precondition failure should remain structured");
|
||||
|
||||
assert_eq!(err, Error::PreconditionFailed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_pool_result_preserves_prefix_access_denied() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_results(
|
||||
vec![RebalanceDeletePoolResult {
|
||||
pool_idx: 0,
|
||||
result: Err(Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned())),
|
||||
}],
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.expect_err("prefix access denial should remain structured");
|
||||
|
||||
assert_eq!(err, Error::PrefixAccessDenied("bucket".to_owned(), "object".to_owned()));
|
||||
fn rebalance_delete_aggregation_rejects_no_attempted_pools() {
|
||||
let error = resolve_rebalance_delete_from_all_pools_results(Vec::new(), "bucket", "object")
|
||||
.expect_err("an empty attempt set cannot report a successful delete");
|
||||
let Error::Io(error) = error else {
|
||||
panic!("an empty attempt set must remain an internal error");
|
||||
};
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::Other);
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"failed to delete rebalance source object bucket/object: no pools were attempted"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,6 +112,7 @@ mod admin;
|
||||
|
||||
pub struct ErasureSetHealer {
|
||||
admin_task: Option<super::task::HealTask>,
|
||||
unresolved_delete_recovery: bool,
|
||||
storage: Arc<dyn HealStorageAPI>,
|
||||
progress: Arc<RwLock<HealProgress>>,
|
||||
cancel_token: tokio_util::sync::CancellationToken,
|
||||
@@ -386,6 +387,7 @@ impl ErasureSetHealer {
|
||||
) -> Self {
|
||||
Self {
|
||||
admin_task: None,
|
||||
unresolved_delete_recovery: false,
|
||||
storage,
|
||||
progress,
|
||||
cancel_token,
|
||||
@@ -406,6 +408,11 @@ impl ErasureSetHealer {
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_unresolved_delete_recovery(mut self, unresolved: bool) -> Self {
|
||||
self.unresolved_delete_recovery = unresolved;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_replacement_execution(mut self, execution: Option<Arc<super::storage::ReplacementExecution>>) -> Self {
|
||||
self.replacement_execution = execution;
|
||||
self
|
||||
@@ -644,6 +651,22 @@ impl ErasureSetHealer {
|
||||
};
|
||||
|
||||
let state = resume_manager.get_state().await;
|
||||
if state.delete_recovery_rescan_pending
|
||||
&& state.completed_buckets.is_empty()
|
||||
&& state.resume_cursor.is_none()
|
||||
&& state.processed_objects == 0
|
||||
&& state.successful_objects == 0
|
||||
&& state.failed_objects == 0
|
||||
&& state.skipped_objects == 0
|
||||
&& state.skipped_new_versions == 0
|
||||
&& state.skipped_ilm_expired == 0
|
||||
&& state.processed_bytes == 0
|
||||
{
|
||||
// Resume is published before its companion checkpoint reset.
|
||||
// Finish that rewind after a crash before clearing the obligation.
|
||||
checkpoint_manager.reset_after_delete_recovery().await?;
|
||||
resume_manager.set_delete_recovery_rescan_pending(false).await?;
|
||||
}
|
||||
if self.admin_task.is_none()
|
||||
&& (state.retry_count > 0 || state.dangling_delete_retry_not_before.is_some())
|
||||
&& state.completed_buckets.is_empty()
|
||||
@@ -706,6 +729,9 @@ impl ErasureSetHealer {
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
) -> Result<()> {
|
||||
if self.unresolved_delete_recovery && !self.heal_opts.dry_run {
|
||||
resume_manager.set_delete_recovery_rescan_pending(true).await?;
|
||||
}
|
||||
// 1. get current state
|
||||
let state = resume_manager.get_state().await;
|
||||
if self.replacement_task_id.is_some()
|
||||
@@ -812,13 +838,27 @@ impl ErasureSetHealer {
|
||||
resume_manager.mark_counter_unknown().await?;
|
||||
}
|
||||
let mut failed_buckets = 0u64;
|
||||
let mut unresolved_delete_recovery = self.unresolved_delete_recovery;
|
||||
|
||||
// 4. process remaining buckets
|
||||
for (bucket_idx, bucket) in buckets.iter().enumerate().skip(current_bucket_index) {
|
||||
// check if completed
|
||||
if state.completed_buckets.contains(bucket) {
|
||||
checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?;
|
||||
current_object_index = 0;
|
||||
let first_bucket_index = if self.admin_task.is_some() { 0 } else { current_bucket_index };
|
||||
for (bucket_idx, bucket) in buckets.iter().enumerate().skip(first_bucket_index) {
|
||||
if bucket_idx < current_bucket_index || state.completed_buckets.contains(bucket) {
|
||||
// Object progress does not certify hidden delete transactions.
|
||||
// Retry recovery in the admitted incarnation without replaying
|
||||
// payload work already acknowledged by the ordinary cursor.
|
||||
if self.admin_task.is_some() {
|
||||
let incarnation = self.admin_bucket_incarnation(checkpoint_manager, bucket).await?;
|
||||
let unresolved = self.heal_admin_bucket_delete_recovery(bucket, incarnation).await?;
|
||||
if unresolved && !self.heal_opts.dry_run {
|
||||
resume_manager.set_delete_recovery_rescan_pending(true).await?;
|
||||
}
|
||||
unresolved_delete_recovery |= unresolved;
|
||||
}
|
||||
if bucket_idx >= current_bucket_index {
|
||||
checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?;
|
||||
current_object_index = 0;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -843,7 +883,7 @@ impl ErasureSetHealer {
|
||||
.await;
|
||||
|
||||
if matches!(bucket_result, Err(Error::TaskCancelled | Error::TaskTimeout)) {
|
||||
return bucket_result;
|
||||
return bucket_result.map(|_| ());
|
||||
}
|
||||
|
||||
// update progress
|
||||
@@ -899,7 +939,8 @@ impl ErasureSetHealer {
|
||||
|
||||
// process bucket result
|
||||
match bucket_result {
|
||||
Ok(_) => {
|
||||
Ok(unresolved) => {
|
||||
unresolved_delete_recovery |= unresolved;
|
||||
resume_manager.complete_bucket(bucket).await?;
|
||||
checkpoint_manager.complete_bucket(bucket_idx.saturating_add(1)).await?;
|
||||
debug!(
|
||||
@@ -970,8 +1011,24 @@ impl ErasureSetHealer {
|
||||
.await?;
|
||||
}
|
||||
|
||||
if !unresolved_delete_recovery && resume_manager.get_state().await.delete_recovery_rescan_pending {
|
||||
// Recovery can publish formerly hidden objects, including when a
|
||||
// background worker settled and collected its proof before this
|
||||
// retry. Aggregate counters only support a whole-set rewind.
|
||||
resume_manager.restart_scan_after_delete_recovery().await?;
|
||||
checkpoint_manager.reset_after_delete_recovery().await?;
|
||||
resume_manager.set_delete_recovery_rescan_pending(false).await?;
|
||||
return Err(Error::transient_skip("Delete recovery settled; ordinary coverage must be rescanned"));
|
||||
}
|
||||
|
||||
if self.admin_task.is_some() {
|
||||
return Ok(());
|
||||
return if unresolved_delete_recovery {
|
||||
Err(Error::transient_skip(
|
||||
"Delete recovery remains unresolved after ordinary erasure-set heal",
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
}
|
||||
|
||||
// 5. finalize. Only declare the set healed when nothing failed AND
|
||||
@@ -1081,6 +1138,12 @@ impl ErasureSetHealer {
|
||||
)));
|
||||
}
|
||||
|
||||
if unresolved_delete_recovery {
|
||||
return Err(Error::transient_skip(
|
||||
"Delete recovery remains unresolved after ordinary erasure-set heal",
|
||||
));
|
||||
}
|
||||
|
||||
// No failures — ordinary heals are complete now. Replacement heals
|
||||
// atomically transition to Verified after the terminal identity fence.
|
||||
if self.replacement_task_id.is_none() {
|
||||
@@ -1353,7 +1416,40 @@ impl ErasureSetHealer {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// heal single bucket with resume
|
||||
async fn heal_admin_bucket_delete_recovery(&self, bucket: &str, incarnation: Option<uuid::Uuid>) -> Result<bool> {
|
||||
let Some(task) = &self.admin_task else {
|
||||
return Ok(false);
|
||||
};
|
||||
let bucket_opts = HealOpts {
|
||||
recursive: false,
|
||||
remove: false,
|
||||
..self.heal_opts
|
||||
};
|
||||
let result = match incarnation {
|
||||
Some(expected) => {
|
||||
self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
|
||||
self.storage
|
||||
.heal_bucket_at_incarnation(bucket, expected, &bucket_opts)
|
||||
.await?
|
||||
}
|
||||
None => self.storage.heal_bucket(bucket, &bucket_opts).await?,
|
||||
};
|
||||
task.record_result_item(result).await;
|
||||
// Recovery retains the checkpoint's admitted incarnation, and the
|
||||
// publication bucket must exist before adopting survivor delete proof.
|
||||
if let Some(result) = self
|
||||
.storage
|
||||
.recover_delete_transactions(bucket, None, incarnation, &self.heal_opts)
|
||||
.await?
|
||||
{
|
||||
let unresolved = result.delete_recovery.is_some_and(|status| status.has_unresolved());
|
||||
task.record_result_item(result).await;
|
||||
return Ok(unresolved);
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Heal one bucket's ordinary objects and report unresolved delete recovery separately.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
|
||||
async fn heal_bucket_with_resume(
|
||||
@@ -1369,7 +1465,7 @@ impl ErasureSetHealer {
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
started_at_secs: u64,
|
||||
) -> Result<()> {
|
||||
) -> Result<bool> {
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
event = EVENT_HEAL_ERASURE_BUCKET_STATE,
|
||||
@@ -1384,31 +1480,9 @@ impl ErasureSetHealer {
|
||||
);
|
||||
|
||||
let incarnation = self.admin_bucket_incarnation(checkpoint_manager, bucket).await?;
|
||||
if let Some(task) = &self.admin_task {
|
||||
let bucket_opts = HealOpts {
|
||||
recursive: false,
|
||||
remove: false,
|
||||
..self.heal_opts
|
||||
};
|
||||
let result = match incarnation {
|
||||
Some(expected) => {
|
||||
self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
|
||||
self.storage
|
||||
.heal_bucket_at_incarnation(bucket, expected, &bucket_opts)
|
||||
.await?
|
||||
}
|
||||
None => self.storage.heal_bucket(bucket, &bucket_opts).await?,
|
||||
};
|
||||
task.record_result_item(result).await;
|
||||
// Keep recovery in the checkpoint's admitted incarnation and repair
|
||||
// the publication bucket before adopting a survivor's delete proof.
|
||||
if let Some(result) = self
|
||||
.storage
|
||||
.recover_delete_transactions(bucket, None, incarnation, &self.heal_opts)
|
||||
.await?
|
||||
{
|
||||
task.record_result_item(result).await;
|
||||
}
|
||||
let unresolved_delete_recovery = self.heal_admin_bucket_delete_recovery(bucket, incarnation).await?;
|
||||
if unresolved_delete_recovery && !self.heal_opts.dry_run {
|
||||
resume_manager.set_delete_recovery_rescan_pending(true).await?;
|
||||
}
|
||||
|
||||
// 1. get bucket info
|
||||
@@ -1425,7 +1499,7 @@ impl ErasureSetHealer {
|
||||
state = "missing",
|
||||
"Erasure set bucket heal skipped because bucket is missing"
|
||||
);
|
||||
return Ok(());
|
||||
return Ok(unresolved_delete_recovery);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1946,7 +2020,7 @@ impl ErasureSetHealer {
|
||||
previous_page_last = page_last;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
Ok(unresolved_delete_recovery)
|
||||
}
|
||||
|
||||
/// initialize progress tracking
|
||||
@@ -2818,7 +2892,7 @@ mod resume_loop_tests {
|
||||
state.start_time,
|
||||
)
|
||||
.await;
|
||||
(processed, successful, failed, skipped, result)
|
||||
(processed, successful, failed, skipped, result.map(|_| ()))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -3888,6 +3962,112 @@ mod resume_loop_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unresolved_delete_recovery_rechecks_coverage_only_after_settlement() {
|
||||
let mut env = make_env().await;
|
||||
env.healer.unresolved_delete_recovery = true;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("healthy", None, false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
|
||||
let error = env
|
||||
.healer
|
||||
.execute_heal_with_resume(&["b".to_owned()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect_err("unresolved recovery cannot publish set completion");
|
||||
assert!(matches!(error, Error::TransientSkip { .. }));
|
||||
assert_eq!(env.storage.calls(), [("healthy".to_owned(), None)]);
|
||||
assert!(!env.resume.get_state().await.completed);
|
||||
assert!(CheckpointManager::has_checkpoint(&env.healer.disk, &env.task_id).await);
|
||||
env.healer
|
||||
.execute_heal_with_resume(&["b".to_owned()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect_err("pending-only retries remain unresolved");
|
||||
assert_eq!(
|
||||
env.storage.calls(),
|
||||
[("healthy".to_owned(), None)],
|
||||
"pending-only retries do not repeat payload scans"
|
||||
);
|
||||
|
||||
env.healer.unresolved_delete_recovery = false;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("healthy", None, false), item("restored", None, false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.healer
|
||||
.execute_heal_with_resume(&["b".to_owned()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect_err("settlement invalidates coverage even if another worker already collected the transaction");
|
||||
assert_eq!(env.storage.calls(), [("healthy".to_owned(), None)]);
|
||||
assert_eq!(
|
||||
env.resume.get_state().await.retry_count,
|
||||
0,
|
||||
"coverage invalidation does not spend failure retries"
|
||||
);
|
||||
env.healer
|
||||
.execute_heal_with_resume(&["b".to_owned()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect("fresh coverage includes the restored object");
|
||||
assert!(env.resume.get_state().await.completed);
|
||||
assert_eq!(env.storage.calls().iter().filter(|(object, _)| object == "restored").count(), 1);
|
||||
assert_eq!(env.resume.get_state().await.processed_objects, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_coverage_rewind_survives_interrupted_checkpoint_reset() {
|
||||
let mut env = make_env().await;
|
||||
env.healer.unresolved_delete_recovery = true;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("healthy", None, false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.healer
|
||||
.execute_heal_with_resume(&["b".to_owned()], "pool_0_set_0", &env.resume, &env.checkpoint)
|
||||
.await
|
||||
.expect_err("create overlapping coverage");
|
||||
assert_eq!(env.checkpoint.get_checkpoint().await.current_bucket_index, 1);
|
||||
env.resume
|
||||
.restart_scan_after_delete_recovery()
|
||||
.await
|
||||
.expect("publish rewind before its companion checkpoint");
|
||||
|
||||
let (resume, checkpoint) = env
|
||||
.healer
|
||||
.initialize_resume_state(&env.task_id, "pool_0_set_0", &["b".to_owned()])
|
||||
.await
|
||||
.expect("restart must finish the interrupted checkpoint rewind");
|
||||
assert!(!resume.get_state().await.delete_recovery_rescan_pending);
|
||||
assert_eq!(checkpoint.get_checkpoint().await.current_bucket_index, 0);
|
||||
env.healer.unresolved_delete_recovery = false;
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![item("healthy", None, false), item("restored", None, false)],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
env.healer
|
||||
.execute_heal_with_resume(&["b".to_owned()], "pool_0_set_0", &resume, &checkpoint)
|
||||
.await
|
||||
.expect("restarted coverage must visit newly visible objects");
|
||||
assert!(resume.get_state().await.completed);
|
||||
assert_eq!(env.storage.calls().iter().filter(|(object, _)| object == "restored").count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_transient_skip_pass_not_marked_completed_state_survives() {
|
||||
// The finalize block must NOT declare a clean completion when a pass had
|
||||
|
||||
@@ -1873,8 +1873,10 @@ async fn root_recovery_legacy_cluster_requires_a_fresh_generation_bound_task() {
|
||||
|
||||
let restarted = recovery_manager(vec![disk]);
|
||||
restarted.replay_root_heals().await.expect("replay legacy root record");
|
||||
let queued = restarted.heal_queue.lock().await.requests().cloned().collect::<Vec<_>>();
|
||||
assert!(queued.is_empty(), "unfenced legacy work must not bind to a later bucket generation");
|
||||
assert!(
|
||||
restarted.heal_queue.lock().await.requests().next().is_none(),
|
||||
"unfenced legacy work must not bind to a later bucket generation"
|
||||
);
|
||||
assert!(
|
||||
restarted
|
||||
.root_recovery
|
||||
|
||||
@@ -321,6 +321,10 @@ pub struct ResumeState {
|
||||
/// the version listing. `None` means "start from the beginning".
|
||||
#[serde(default)]
|
||||
pub resume_cursor: Option<String>,
|
||||
/// Ordinary coverage overlapped unresolved hidden delete recovery. The
|
||||
/// scan must restart after recovery settles, even if another worker settled it.
|
||||
#[serde(default)]
|
||||
pub delete_recovery_rescan_pending: bool,
|
||||
/// task id
|
||||
pub task_id: String,
|
||||
/// task type
|
||||
@@ -364,6 +368,11 @@ pub struct ResumeState {
|
||||
/// and may receive one bounded compatibility re-arm.
|
||||
#[serde(default)]
|
||||
pub replacement_legacy_retry_compatibility_done: bool,
|
||||
/// New generations cannot consume the one-shot compatibility handoff for
|
||||
/// a pre-format intent stranded by historical rebalance topology. Missing
|
||||
/// fields in older schema-seven snapshots remain eligible for validation.
|
||||
#[serde(default)]
|
||||
pub replacement_preformat_handoff_retry_done: bool,
|
||||
/// The current pass was rewound because the replacement mount was not
|
||||
/// ready. Task-level error persistence must not turn this readiness wait
|
||||
/// back into a consumed retry attempt.
|
||||
@@ -434,6 +443,7 @@ impl ResumeState {
|
||||
Self {
|
||||
schema_version: CURRENT_RESUME_SCHEMA,
|
||||
resume_cursor: None,
|
||||
delete_recovery_rescan_pending: false,
|
||||
task_id,
|
||||
task_type,
|
||||
set_disk_id,
|
||||
@@ -450,6 +460,7 @@ impl ResumeState {
|
||||
replacement_legacy_import: None,
|
||||
replacement_legacy_successor: None,
|
||||
replacement_legacy_retry_compatibility_done: false,
|
||||
replacement_preformat_handoff_retry_done: false,
|
||||
replacement_retry_waiting_for_target: false,
|
||||
dangling_delete_retry_not_before: None,
|
||||
start_time: SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs(),
|
||||
@@ -495,6 +506,7 @@ impl ResumeState {
|
||||
// New generations already use readiness-aware persistence and must
|
||||
// never enter the one-shot compatibility path for legacy snapshots.
|
||||
state.replacement_legacy_retry_compatibility_done = true;
|
||||
state.replacement_preformat_handoff_retry_done = true;
|
||||
state
|
||||
}
|
||||
|
||||
@@ -1048,6 +1060,30 @@ impl ResumeManager {
|
||||
self.state.read().await.clone()
|
||||
}
|
||||
|
||||
pub(crate) async fn set_delete_recovery_rescan_pending(&self, pending: bool) -> Result<()> {
|
||||
let mut state = self.state.write().await;
|
||||
let previous = state.delete_recovery_rescan_pending;
|
||||
if previous == pending {
|
||||
return Ok(());
|
||||
}
|
||||
state.delete_recovery_rescan_pending = pending;
|
||||
drop(state);
|
||||
if let Err(error) = self.save_state_strict().await {
|
||||
self.state.write().await.delete_recovery_rescan_pending = previous;
|
||||
return Err(error);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Keep the coverage obligation until the companion checkpoint rewind is durable.
|
||||
pub(crate) async fn restart_scan_after_delete_recovery(&self) -> Result<()> {
|
||||
let mut state = self.state.write().await;
|
||||
state.delete_recovery_rescan_pending = true;
|
||||
state.reset_for_retry();
|
||||
drop(state);
|
||||
self.save_state_strict().await
|
||||
}
|
||||
|
||||
/// update progress
|
||||
pub async fn update_progress(&self, processed: u64, successful: u64, failed: u64, skipped: u64) -> Result<()> {
|
||||
let mut state = self.state.write().await;
|
||||
@@ -1243,6 +1279,9 @@ impl ResumeManager {
|
||||
if state.max_retries == 0 {
|
||||
return Ok(false);
|
||||
}
|
||||
if self.rearm_preformat_handoff_retry_if_needed(storage).await? {
|
||||
return Ok(true);
|
||||
}
|
||||
let legacy_rearm = replacement_retry_is_legacy_transient_skip(&state);
|
||||
let ready = storage
|
||||
.replacement_targets_ready_for_retry(
|
||||
|
||||
@@ -602,6 +602,18 @@ impl CheckpointManager {
|
||||
self.save_checkpoint_throttled().await
|
||||
}
|
||||
|
||||
pub(crate) async fn reset_after_delete_recovery(&self) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
checkpoint.reset_for_retry();
|
||||
if let Some(admin) = checkpoint.admin.as_mut() {
|
||||
admin.completed = false;
|
||||
admin.outcome = Default::default();
|
||||
admin.outcome.start();
|
||||
}
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint().await
|
||||
}
|
||||
|
||||
/// Add a processed object. Called once per healed object, so persistence
|
||||
/// is batched (`PERSIST_EVERY_MUTATIONS` / `PERSIST_INTERVAL`); positions
|
||||
/// and page boundaries still persist unconditionally.
|
||||
|
||||
@@ -89,6 +89,65 @@ pub(super) fn validate_lineage(task_id: &str, set_disk_id: &str, lineage: &[Repl
|
||||
}
|
||||
|
||||
impl ResumeManager {
|
||||
// RUSTFS_COMPAT_TODO(8565): recover pre-format intents stranded by historical pool topology. Remove after supported upgrades no longer admit snapshots lacking replacement_preformat_handoff_retry_done and retained affected intents are resolved.
|
||||
pub(super) async fn rearm_preformat_handoff_retry_if_needed(
|
||||
&self,
|
||||
storage: &dyn crate::heal::storage::HealStorageAPI,
|
||||
) -> Result<bool> {
|
||||
let state = self.get_state().await;
|
||||
if !preformat_handoff_retry_candidate(&state) {
|
||||
return Ok(false);
|
||||
}
|
||||
let identities = storage.replacement_target_identities(&state.replacement_targets).await?;
|
||||
if identities == state.replacement_target_identities {
|
||||
return Ok(false);
|
||||
}
|
||||
// Lock order: target execution leases, the store's format fence during
|
||||
// the probe, then the intent CAS after the probe releases its fence.
|
||||
let execution = storage.replacement_execution(&state.replacement_targets).await?;
|
||||
if execution.identities() != identities.as_slice() {
|
||||
return Err(Error::ReplacementTargetNotReady(
|
||||
"target changed before pre-format recovery admission".to_string(),
|
||||
));
|
||||
}
|
||||
// A cleared summary does not prove an unstarted pass. Any checkpoint,
|
||||
// including an empty/corrupt file or an isolated checkpoint marker,
|
||||
// prevents this compatibility admission.
|
||||
for suffix in [RESUME_CHECKPOINT_FILE, RESUME_CHECKPOINT_BLOCKED_FILE] {
|
||||
let path = Path::new(BUCKET_META_PREFIX).join(format!("{}_{suffix}", state.task_id));
|
||||
match self.disk.read_all(RUSTFS_META_BUCKET, path_to_str(&path)?).await {
|
||||
Ok(_) => return Ok(false),
|
||||
Err(DiskError::FileNotFound) => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
if Self::replacement_completion_proof_if_present(self.disk.clone(), &state.task_id)
|
||||
.await?
|
||||
.is_some()
|
||||
|| !storage
|
||||
.replacement_preformat_retry_is_safe(&state.set_disk_id, &state.replacement_targets)
|
||||
.await?
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let buckets = storage.list_buckets().await?.into_iter().map(|bucket| bucket.name).collect();
|
||||
if storage
|
||||
.replacement_target_identities(&state.replacement_targets)
|
||||
.await?
|
||||
.as_slice()
|
||||
!= execution.identities()
|
||||
{
|
||||
return Err(Error::ReplacementTargetNotReady(
|
||||
"target changed during pre-format recovery admission".to_string(),
|
||||
));
|
||||
}
|
||||
// Retain the execution leases through validation and the Prepared CAS.
|
||||
// Budget admission, its one-shot marker, and its reserved successor are
|
||||
// one durable decision, so a crash cannot grant an unbound retry.
|
||||
self.prepare_replacement_handoff_inner(&execution, buckets, true).await?;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Shared startup/scanner decision. A changed mount starts a full scan in
|
||||
/// one durable successor; an unfinished transfer always reuses that UUID.
|
||||
pub(crate) async fn resolve_replacement_recovery(
|
||||
@@ -184,11 +243,23 @@ impl ResumeManager {
|
||||
/// Select a successor only after every old executor is fenced by the target
|
||||
/// execution leases. The durable predecessor is the sole handoff authority.
|
||||
pub(crate) async fn prepare_replacement_handoff(
|
||||
&self,
|
||||
execution: &ReplacementExecution,
|
||||
buckets: Vec<String>,
|
||||
) -> Result<ResumeManager> {
|
||||
self.prepare_replacement_handoff_inner(execution, buckets, false).await
|
||||
}
|
||||
|
||||
async fn prepare_replacement_handoff_inner(
|
||||
&self,
|
||||
execution: &ReplacementExecution,
|
||||
mut buckets: Vec<String>,
|
||||
preformat_retry: bool,
|
||||
) -> Result<ResumeManager> {
|
||||
let state = self.get_state().await;
|
||||
if preformat_retry && !preformat_handoff_retry_candidate(&state) {
|
||||
return Err(replacement_recovery_conflict("pre-format recovery eligibility changed"));
|
||||
}
|
||||
if let Some(handoff) = &state.replacement_handoff {
|
||||
Self::validate_handoff(handoff, &state)?;
|
||||
let raw = Self::read_state_file(&self.disk, &state.task_id, self.state_file).await?;
|
||||
@@ -219,7 +290,7 @@ impl ResumeManager {
|
||||
"replacement generation is not eligible for an online handoff",
|
||||
));
|
||||
}
|
||||
if state.retry_count.saturating_add(1) >= state.max_retries {
|
||||
if !preformat_retry && state.retry_count.saturating_add(1) >= state.max_retries {
|
||||
return Err(Error::ReplacementRetryBudgetExhausted);
|
||||
}
|
||||
if state.replacement_phase == ReplacementPhase::OwnershipPending {
|
||||
@@ -280,6 +351,12 @@ impl ResumeManager {
|
||||
if current.replacement_revision != state.replacement_revision {
|
||||
return Err(replacement_recovery_conflict("replacement changed during handoff preparation"));
|
||||
}
|
||||
if preformat_retry {
|
||||
current.replacement_preformat_handoff_retry_done = true;
|
||||
// One handoff consumes one attempt; its successor must still
|
||||
// have exactly one actual rebuild attempt under the same cap.
|
||||
current.retry_count = current.max_retries - 2;
|
||||
}
|
||||
current.replacement_handoff = Some(ReplacementHandoff {
|
||||
phase: ReplacementHandoffPhase::Prepared,
|
||||
link,
|
||||
@@ -378,6 +455,8 @@ impl ResumeManager {
|
||||
link.targets.clone(),
|
||||
);
|
||||
state.replacement_phase = ReplacementPhase::OwnershipPending;
|
||||
// The constructor marks this compatibility grant consumed for every
|
||||
// successor, including descendants of a recovered older generation.
|
||||
state.replacement_predecessor = Some(predecessor.task_id.clone());
|
||||
state.retry_count = predecessor
|
||||
.retry_count
|
||||
@@ -493,6 +572,31 @@ impl ResumeManager {
|
||||
}
|
||||
}
|
||||
|
||||
fn preformat_handoff_retry_candidate(state: &ResumeState) -> bool {
|
||||
replacement_retry_is_exhausted_active(state)
|
||||
&& state.replacement_generation.as_deref() == Some(state.task_id.as_str())
|
||||
&& !state.replacement_preformat_handoff_retry_done
|
||||
&& state.replacement_phase == ReplacementPhase::Intent
|
||||
&& state.replacement_execution_protocol == 1
|
||||
&& state.max_retries >= 2
|
||||
&& state.replacement_predecessor.is_none()
|
||||
&& state.replacement_handoff.is_none()
|
||||
&& state.replacement_lineage.is_empty()
|
||||
&& state.replacement_legacy_import.is_none()
|
||||
&& state.replacement_legacy_successor.is_none()
|
||||
&& state.resume_cursor.is_none()
|
||||
&& state.current_bucket.is_none()
|
||||
&& state.current_object.is_none()
|
||||
&& state.completed_buckets.is_empty()
|
||||
&& state.processed_objects == 0
|
||||
&& state.successful_objects == 0
|
||||
&& state.failed_objects == 0
|
||||
&& state.skipped_objects == 0
|
||||
&& state.skipped_new_versions == 0
|
||||
&& state.skipped_ilm_expired == 0
|
||||
&& state.processed_bytes == 0
|
||||
}
|
||||
|
||||
fn successor_can_be_rebound(state: &ResumeState, predecessor: &str) -> bool {
|
||||
state.replacement_predecessor.as_deref() == Some(predecessor) && successor_has_no_progress(state)
|
||||
}
|
||||
|
||||
@@ -13,10 +13,451 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::heal::{HEALING_MARKER_PATH, resume::tests::schema_test_disk};
|
||||
use std::sync::LazyLock;
|
||||
use crate::heal::{
|
||||
HEALING_MARKER_PATH, HealObjectInfo,
|
||||
resume::tests::schema_test_disk,
|
||||
storage::{HealListItem, HealStorageAPI},
|
||||
storage_api::storage::{BucketInfo, DiskSetSelector},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::sync::{
|
||||
LazyLock,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
static FAILURES: LazyLock<Mutex<HashMap<String, &'static str>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
struct PreformatRecoveryStorage {
|
||||
execution: Arc<ReplacementExecution>,
|
||||
historical_topology: bool,
|
||||
probe_calls: AtomicUsize,
|
||||
identity_calls: AtomicUsize,
|
||||
change_identity_on_call: usize,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl HealStorageAPI for PreformatRecoveryStorage {
|
||||
async fn replacement_target_identities(&self, _: &[String]) -> Result<Vec<ReplacementTargetIdentity>> {
|
||||
let mut identities = self.execution.identities().to_vec();
|
||||
if self.identity_calls.fetch_add(1, Ordering::SeqCst) == self.change_identity_on_call {
|
||||
identities[0].filesystem_identity = "another-mount:fs:root".to_string();
|
||||
}
|
||||
Ok(identities)
|
||||
}
|
||||
|
||||
async fn replacement_execution(&self, _: &[String]) -> Result<Arc<ReplacementExecution>> {
|
||||
Ok(self.execution.clone())
|
||||
}
|
||||
|
||||
async fn replacement_targets_ready_for_retry(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &[String],
|
||||
expected: &[ReplacementTargetIdentity],
|
||||
) -> Result<bool> {
|
||||
if expected != self.execution.identities() {
|
||||
return Ok(false);
|
||||
}
|
||||
for disk in self.execution.test_disks() {
|
||||
if !matches!(disk.get_disk_id().await, Ok(Some(id)) if !id.is_nil()) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn replacement_preformat_retry_is_safe(&self, _: &str, _: &[String]) -> Result<bool> {
|
||||
self.probe_calls.fetch_add(1, Ordering::SeqCst);
|
||||
if !self.historical_topology {
|
||||
return Ok(false);
|
||||
}
|
||||
for disk in self.execution.test_disks() {
|
||||
match disk.read_all(RUSTFS_META_BUCKET, "format.json").await {
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
|
||||
Ok(_) => return Ok(false),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
async fn get_object_meta(&self, _: &str, _: &str) -> Result<Option<HealObjectInfo>> {
|
||||
Err(Error::other("pre-format recovery must not inspect objects"))
|
||||
}
|
||||
|
||||
async fn ec_decode_rebuild(&self, _: &str, _: &str) -> Result<Vec<u8>> {
|
||||
Err(Error::other("pre-format recovery must not rebuild objects"))
|
||||
}
|
||||
|
||||
async fn get_bucket_info(&self, _: &str) -> Result<Option<BucketInfo>> {
|
||||
Err(Error::other("pre-format recovery must not inspect buckets"))
|
||||
}
|
||||
|
||||
async fn list_buckets_for_admin_heal(&self, _: Option<DiskSetSelector>) -> Result<Vec<BucketInfo>> {
|
||||
Err(Error::other("pre-format recovery must not start admin discovery"))
|
||||
}
|
||||
|
||||
async fn object_exists(&self, _: &str, _: &str) -> Result<bool> {
|
||||
Err(Error::other("pre-format recovery must not inspect objects"))
|
||||
}
|
||||
|
||||
async fn heal_object(&self, _: &str, _: &str, _: Option<&str>, _: &HealOpts) -> Result<(HealResultItem, Option<Error>)> {
|
||||
Err(Error::other("pre-format recovery must not heal objects"))
|
||||
}
|
||||
|
||||
async fn heal_bucket(&self, _: &str, _: &HealOpts) -> Result<HealResultItem> {
|
||||
Err(Error::other("pre-format recovery must not heal buckets"))
|
||||
}
|
||||
|
||||
async fn heal_format(&self, _: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
Err(Error::other("pre-format recovery must not write formats"))
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &str,
|
||||
_: Option<&str>,
|
||||
_: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Err(Error::other("pre-format recovery must not traverse objects"))
|
||||
}
|
||||
|
||||
async fn get_disk_for_resume(&self, _: &str) -> Result<DiskStore> {
|
||||
Err(Error::other("pre-format recovery must retain its durable anchor"))
|
||||
}
|
||||
}
|
||||
|
||||
async fn preformat_recovery_fixture() -> (Vec<tempfile::TempDir>, ResumeManager, PreformatRecoveryStorage) {
|
||||
let (dirs, manager, execution) = fixture(1).await;
|
||||
execution.test_disks()[0]
|
||||
.delete(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Default::default())
|
||||
.await
|
||||
.expect("remove the fixture marker from the blank replacement");
|
||||
{
|
||||
let mut state = manager.state.write().await;
|
||||
state.retry_count = state.max_retries;
|
||||
state.replacement_preformat_handoff_retry_done = false;
|
||||
state.replacement_retry_waiting_for_target = true;
|
||||
}
|
||||
manager.save_state_strict().await.expect("persist exhausted older intent");
|
||||
(
|
||||
dirs,
|
||||
manager,
|
||||
PreformatRecoveryStorage {
|
||||
execution,
|
||||
historical_topology: true,
|
||||
probe_calls: AtomicUsize::new(0),
|
||||
identity_calls: AtomicUsize::new(0),
|
||||
change_identity_on_call: usize::MAX,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preformat_recovery_same_mount_reuses_existing_readiness_rearm() {
|
||||
let (_dirs, manager, mut storage) = preformat_recovery_fixture().await;
|
||||
let initial = manager.get_state().await;
|
||||
storage.execution =
|
||||
ReplacementExecution::for_test(storage.execution.test_disks().to_vec(), initial.replacement_target_identities.clone());
|
||||
assert!(matches!(
|
||||
storage.execution.test_disks()[0].get_disk_id().await,
|
||||
Err(DiskError::UnformattedDisk)
|
||||
));
|
||||
assert!(
|
||||
manager
|
||||
.rearm_replacement_recovery_if_needed(&storage)
|
||||
.await
|
||||
.expect("rearm blank same mount")
|
||||
);
|
||||
let resumed = manager
|
||||
.resolve_replacement_recovery(&storage)
|
||||
.await
|
||||
.expect("resume the same generation");
|
||||
assert_eq!(resumed.task_id, initial.task_id);
|
||||
assert_eq!(resumed.retry_count, resumed.max_retries - 1);
|
||||
assert!(resumed.replacement_handoff.is_none());
|
||||
assert!(!resumed.replacement_preformat_handoff_retry_done);
|
||||
assert_eq!(storage.probe_calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preformat_recovery_reserves_one_handoff_and_one_rebuild() {
|
||||
let (_dirs, manager, mut storage) = preformat_recovery_fixture().await;
|
||||
assert!(
|
||||
manager
|
||||
.rearm_replacement_recovery_if_needed(&storage)
|
||||
.await
|
||||
.expect("admit historical pre-format recovery")
|
||||
);
|
||||
let prepared = manager.get_state().await;
|
||||
assert_eq!(prepared.replacement_phase, ReplacementPhase::HandoffPending);
|
||||
assert!(prepared.replacement_preformat_handoff_retry_done);
|
||||
let child_state = manager
|
||||
.resolve_replacement_recovery(&storage)
|
||||
.await
|
||||
.expect("resolve the reserved successor");
|
||||
assert_eq!(child_state.retry_count, child_state.max_retries - 1);
|
||||
assert!(child_state.replacement_preformat_handoff_retry_done);
|
||||
assert_eq!(child_state.replacement_lineage.len(), 1);
|
||||
let child = ResumeManager::load_replacement_intent(manager.disk.clone(), &child_state.task_id)
|
||||
.await
|
||||
.expect("reload the admitted successor");
|
||||
// Unlike a generic schema-test disk, an admitted replacement has an exact
|
||||
// set/slot binding that get_disk_id validates against its persisted format.
|
||||
let mut endpoint = storage.execution.test_disks()[0].endpoint();
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let disk = crate::heal::new_disk(
|
||||
&endpoint,
|
||||
&crate::heal::DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("open the replacement with its real erasure-set slot");
|
||||
storage.execution = ReplacementExecution::for_test(vec![disk.clone()], storage.execution.identities().to_vec());
|
||||
let disk_id = Uuid::new_v4();
|
||||
let format = serde_json::json!({
|
||||
"version": "1", "format": "xl", "id": Uuid::new_v4(),
|
||||
"xl": {"version": "3", "this": disk_id, "sets": [[disk_id]], "distributionAlgo": "SIPMOD+PARITY"}
|
||||
});
|
||||
storage.execution.test_disks()[0]
|
||||
.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
"format.json",
|
||||
serde_json::to_vec(&format).expect("serialize target format").into(),
|
||||
)
|
||||
.await
|
||||
.expect("format the admitted target before rebuilding");
|
||||
crate::heal::storage_api::EcstoreDiskAPI::set_disk_id(disk.as_ref(), Some(disk_id))
|
||||
.await
|
||||
.expect("publish the formatted disk identity as save_format_file does");
|
||||
assert_eq!(disk.get_disk_id().await.expect("formatted replacement must be ready"), Some(disk_id));
|
||||
child
|
||||
.acquire_replacement_markers(&storage.execution)
|
||||
.await
|
||||
.expect("commit successor ownership");
|
||||
child
|
||||
.mark_replacement_rebuilding(storage.execution.identities().to_vec())
|
||||
.await
|
||||
.expect("admit actual rebuilding");
|
||||
child
|
||||
.record_replacement_failure(&Error::other("actual rebuild failed"), child_state.max_retries)
|
||||
.await
|
||||
.expect("exhaust final attempt");
|
||||
assert!(
|
||||
!child
|
||||
.rearm_replacement_recovery_if_needed(&storage)
|
||||
.await
|
||||
.expect("never regrant the successor")
|
||||
);
|
||||
let reloaded = ResumeManager::load_replacement_intent(manager.disk.clone(), &prepared.task_id)
|
||||
.await
|
||||
.expect("reload predecessor");
|
||||
assert!(
|
||||
!reloaded
|
||||
.rearm_preformat_handoff_retry_if_needed(&storage)
|
||||
.await
|
||||
.expect("never regrant the predecessor")
|
||||
);
|
||||
assert_eq!(storage.probe_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preformat_recovery_replays_prepared_and_published_crashes_without_new_credit() {
|
||||
for boundary in ["prepared", "successor_published"] {
|
||||
let (_dirs, manager, storage) = preformat_recovery_fixture().await;
|
||||
let source = manager.get_state().await.task_id;
|
||||
FAILURES
|
||||
.lock()
|
||||
.expect("failure injection lock")
|
||||
.insert(source.clone(), boundary);
|
||||
assert!(manager.rearm_replacement_recovery_if_needed(&storage).await.is_err());
|
||||
let reloaded = ResumeManager::load_replacement_intent(manager.disk.clone(), &source)
|
||||
.await
|
||||
.expect("reload crashed grant");
|
||||
let prepared = reloaded.get_state().await;
|
||||
assert!(prepared.replacement_preformat_handoff_retry_done);
|
||||
let successor = prepared
|
||||
.replacement_handoff
|
||||
.expect("grant and successor are atomic")
|
||||
.link
|
||||
.successor;
|
||||
assert!(
|
||||
!reloaded
|
||||
.rearm_preformat_handoff_retry_if_needed(&storage)
|
||||
.await
|
||||
.expect("grant already used")
|
||||
);
|
||||
let recovered = reloaded
|
||||
.resolve_replacement_recovery(&storage)
|
||||
.await
|
||||
.expect("replay the reserved handoff");
|
||||
assert_eq!(recovered.task_id, successor);
|
||||
assert_eq!(recovered.retry_count, recovered.max_retries - 1);
|
||||
assert_eq!(storage.probe_calls.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preformat_recovery_rejects_progress_checkpoints_formats_and_unrelated_topology() {
|
||||
for rejection in [
|
||||
"progress",
|
||||
"checkpoint",
|
||||
"blocked",
|
||||
"completion_proof",
|
||||
"format",
|
||||
"topology",
|
||||
"new_generation",
|
||||
"single_retry",
|
||||
] {
|
||||
let (_dirs, manager, mut storage) = preformat_recovery_fixture().await;
|
||||
let source = manager.get_state().await.task_id;
|
||||
match rejection {
|
||||
"progress" => manager.state.write().await.processed_objects = 1,
|
||||
"checkpoint" | "blocked" => {
|
||||
let suffix = if rejection == "checkpoint" {
|
||||
RESUME_CHECKPOINT_FILE
|
||||
} else {
|
||||
RESUME_CHECKPOINT_BLOCKED_FILE
|
||||
};
|
||||
manager
|
||||
.disk
|
||||
.write_all(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{source}_{suffix}"), Vec::new().into())
|
||||
.await
|
||||
.expect("persist checkpoint evidence, including an empty file");
|
||||
}
|
||||
"format" => storage.execution.test_disks()[0]
|
||||
.write_all(RUSTFS_META_BUCKET, "format.json", b"existing format".to_vec().into())
|
||||
.await
|
||||
.expect("persist actual format evidence"),
|
||||
"completion_proof" => {
|
||||
let proof = super::super::replacement::ReplacementCompletionProof::from_state(&manager.get_state().await, 1)
|
||||
.expect("construct a prior completion proof");
|
||||
manager
|
||||
.disk
|
||||
.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
path_to_str(&replacement_completion_proof_path(&source)).expect("proof path"),
|
||||
serde_json::to_vec(&proof).expect("serialize completion proof").into(),
|
||||
)
|
||||
.await
|
||||
.expect("persist existing completion proof");
|
||||
}
|
||||
"topology" => storage.historical_topology = false,
|
||||
"new_generation" => manager.state.write().await.replacement_preformat_handoff_retry_done = true,
|
||||
"single_retry" => manager.state.write().await.max_retries = 1,
|
||||
_ => unreachable!("known test rejection"),
|
||||
}
|
||||
manager.save_state_strict().await.expect("persist rejection fixture");
|
||||
let before = ResumeManager::read_state_file(&manager.disk, &source, manager.state_file)
|
||||
.await
|
||||
.expect("source bytes before admission");
|
||||
assert!(
|
||||
!manager
|
||||
.rearm_preformat_handoff_retry_if_needed(&storage)
|
||||
.await
|
||||
.expect("reject unsafe recovery"),
|
||||
"{rejection}"
|
||||
);
|
||||
assert_eq!(
|
||||
ResumeManager::read_state_file(&manager.disk, &source, manager.state_file)
|
||||
.await
|
||||
.expect("source bytes after rejection"),
|
||||
before,
|
||||
"{rejection}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preformat_recovery_rejects_mount_changes_before_or_during_probe() {
|
||||
for change_identity_on_call in [0, 1] {
|
||||
let (_dirs, manager, mut storage) = preformat_recovery_fixture().await;
|
||||
storage.change_identity_on_call = change_identity_on_call;
|
||||
let source = manager.get_state().await.task_id;
|
||||
let before = ResumeManager::read_state_file(&manager.disk, &source, manager.state_file)
|
||||
.await
|
||||
.expect("source bytes before mount change");
|
||||
assert!(matches!(
|
||||
manager.rearm_preformat_handoff_retry_if_needed(&storage).await,
|
||||
Err(Error::ReplacementTargetNotReady(_))
|
||||
));
|
||||
assert_eq!(
|
||||
ResumeManager::read_state_file(&manager.disk, &source, manager.state_file)
|
||||
.await
|
||||
.expect("source bytes after mount change"),
|
||||
before
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preformat_recovery_rejects_unknown_marker_and_stale_revision() {
|
||||
let (_dirs, manager, storage) = preformat_recovery_fixture().await;
|
||||
let source = manager.get_state().await.task_id;
|
||||
storage.execution.test_disks()[0]
|
||||
.write_all(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, b"unknown".to_vec().into())
|
||||
.await
|
||||
.expect("publish unknown marker owner");
|
||||
let before = ResumeManager::read_state_file(&manager.disk, &source, manager.state_file)
|
||||
.await
|
||||
.expect("source before owner conflict");
|
||||
assert!(matches!(
|
||||
manager.rearm_preformat_handoff_retry_if_needed(&storage).await,
|
||||
Err(Error::ReplacementOwnershipConflict(_))
|
||||
));
|
||||
assert_eq!(
|
||||
ResumeManager::read_state_file(&manager.disk, &source, manager.state_file)
|
||||
.await
|
||||
.expect("source after owner conflict"),
|
||||
before
|
||||
);
|
||||
storage.execution.test_disks()[0]
|
||||
.delete(RUSTFS_META_BUCKET, HEALING_MARKER_PATH, Default::default())
|
||||
.await
|
||||
.expect("remove fixture owner conflict");
|
||||
let stale = ResumeManager::load_replacement_intent(manager.disk.clone(), &source)
|
||||
.await
|
||||
.expect("open stale source");
|
||||
manager.save_state_strict().await.expect("advance source revision");
|
||||
assert!(stale.rearm_preformat_handoff_retry_if_needed(&storage).await.is_err());
|
||||
let unchanged = ResumeManager::load_replacement_intent(manager.disk.clone(), &source)
|
||||
.await
|
||||
.expect("reload authoritative source")
|
||||
.get_state()
|
||||
.await;
|
||||
assert!(!unchanged.replacement_preformat_handoff_retry_done);
|
||||
assert!(unchanged.replacement_handoff.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preformat_recovery_marker_defaults_only_old_snapshots_to_eligible() {
|
||||
let state = ResumeState::replacement_intent(
|
||||
Uuid::new_v4().to_string(),
|
||||
"erasure_set".into(),
|
||||
"pool_0_set_0".into(),
|
||||
Vec::new(),
|
||||
vec!["replacement-0".into()],
|
||||
vec![identity("replacement-0", "mount:fs:root")],
|
||||
);
|
||||
assert!(state.replacement_preformat_handoff_retry_done);
|
||||
let mut wire = serde_json::to_value(state).expect("serialize new intent");
|
||||
wire.as_object_mut()
|
||||
.expect("intent object")
|
||||
.remove("replacement_preformat_handoff_retry_done");
|
||||
let old: ResumeState = serde_json::from_value(wire).expect("deserialize older schema-seven intent");
|
||||
assert!(!old.replacement_preformat_handoff_retry_done);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handoff_rebinding_never_resets_an_exhausted_successor_budget() {
|
||||
let (_dirs, parent, execution) = fixture(1).await;
|
||||
|
||||
@@ -736,6 +736,13 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Admit a one-shot recovery of an old pre-format intent only when the
|
||||
/// current format fence permits its set, rebalance has a quiescent older
|
||||
/// topology, and every target is a blank, unformatted replacement.
|
||||
async fn replacement_preformat_retry_is_safe(&self, _set_disk_id: &str, _targets: &[String]) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn replacement_execution(&self, _targets: &[String]) -> Result<Arc<ReplacementExecution>> {
|
||||
Err(Error::other("replacement execution lease acquisition is unsupported"))
|
||||
}
|
||||
@@ -2109,6 +2116,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
async fn replacement_execution(&self, targets: &[String]) -> Result<Arc<ReplacementExecution>> {
|
||||
ReplacementExecution::acquire(targets).await
|
||||
}
|
||||
|
||||
async fn replacement_preformat_retry_is_safe(&self, set_disk_id: &str, targets: &[String]) -> Result<bool> {
|
||||
let (pool_index, set_index) = crate::heal::utils::parse_set_disk_id(set_disk_id)?;
|
||||
self.ecstore
|
||||
.replacement_preformat_retry_is_safe(pool_index, set_index, targets)
|
||||
.await
|
||||
.map_err(Error::Storage)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -132,7 +132,7 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio
|
||||
}
|
||||
|
||||
impl HealTask {
|
||||
async fn recover_bucket_delete_transactions(&self, bucket: &str, opts: &HealOpts) -> Result<()> {
|
||||
async fn recover_bucket_delete_transactions(&self, bucket: &str, opts: &HealOpts) -> Result<bool> {
|
||||
if let Some(result) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
@@ -140,9 +140,11 @@ impl HealTask {
|
||||
)
|
||||
.await?
|
||||
{
|
||||
let unresolved = result.delete_recovery.is_some_and(|status| status.has_unresolved());
|
||||
self.record_result_item(result).await;
|
||||
return Ok(unresolved);
|
||||
}
|
||||
Ok(())
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
|
||||
@@ -241,8 +243,8 @@ impl HealTask {
|
||||
};
|
||||
// Removal must settle survivor journals before deleting an empty
|
||||
// replica's publication directory; repair must establish it first.
|
||||
if heal_opts.remove {
|
||||
self.recover_bucket_delete_transactions(bucket, &recovery_opts).await?;
|
||||
if heal_opts.remove && self.recover_bucket_delete_transactions(bucket, &recovery_opts).await? {
|
||||
return Err(Error::transient_skip("Delete recovery must resolve before bucket removal"));
|
||||
}
|
||||
let heal_result = match self.bucket_incarnation_id {
|
||||
Some(expected) => {
|
||||
@@ -269,13 +271,15 @@ impl HealTask {
|
||||
);
|
||||
self.record_result_item(result).await;
|
||||
|
||||
if !heal_opts.remove {
|
||||
self.recover_bucket_delete_transactions(bucket, &recovery_opts).await?;
|
||||
}
|
||||
let unresolved_delete_recovery =
|
||||
!heal_opts.remove && self.recover_bucket_delete_transactions(bucket, &recovery_opts).await?;
|
||||
|
||||
if self.options.recursive {
|
||||
self.heal_bucket_objects(bucket, "").await?;
|
||||
}
|
||||
if unresolved_delete_recovery {
|
||||
return Err(Error::transient_skip("Delete recovery remains unresolved after ordinary bucket heal"));
|
||||
}
|
||||
|
||||
if !self.options.recursive {
|
||||
let mut progress = self.progress.write().await;
|
||||
@@ -563,6 +567,7 @@ impl HealTask {
|
||||
set: self.options.set_index,
|
||||
..Default::default()
|
||||
};
|
||||
let mut unresolved_delete_recovery = false;
|
||||
if let Some(result) = self
|
||||
.await_with_control(self.storage.recover_delete_prefix_transactions(
|
||||
bucket,
|
||||
@@ -572,10 +577,15 @@ impl HealTask {
|
||||
))
|
||||
.await?
|
||||
{
|
||||
unresolved_delete_recovery = result.delete_recovery.is_some_and(|status| status.has_unresolved());
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
|
||||
self.heal_bucket_objects(bucket, prefix).await
|
||||
self.heal_bucket_objects(bucket, prefix).await?;
|
||||
if unresolved_delete_recovery {
|
||||
return Err(Error::transient_skip("Delete recovery remains unresolved after ordinary prefix heal"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[hotpath::measure]
|
||||
|
||||
@@ -460,6 +460,7 @@ impl HealTask {
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
let mut unresolved_delete_recovery = false;
|
||||
for bucket in buckets.iter().filter(|_| self.source != HealRequestSource::Admin) {
|
||||
// Check control flags before starting each bucket heal
|
||||
self.check_control_flags().await?;
|
||||
@@ -498,6 +499,7 @@ impl HealTask {
|
||||
)
|
||||
.await?
|
||||
{
|
||||
unresolved_delete_recovery |= result.delete_recovery.is_some_and(|status| status.has_unresolved());
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
}
|
||||
@@ -521,6 +523,7 @@ impl HealTask {
|
||||
heal_opts,
|
||||
self.source,
|
||||
)
|
||||
.with_unresolved_delete_recovery(unresolved_delete_recovery)
|
||||
.with_replacement_targets(replacement_targets, is_auto_replacement.then(|| self.id.clone()))
|
||||
.with_pool_metadata_targets(if self.options.recreate_missing && !self.options.dry_run {
|
||||
self.heal_endpoints.clone()
|
||||
|
||||
@@ -1537,6 +1537,8 @@ struct MockStorage {
|
||||
bucket_discovery_incomplete: bool,
|
||||
bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
|
||||
bucket_heal_calls: Mutex<Vec<String>>,
|
||||
delete_recovery_results: Mutex<HashMap<String, Result<Option<HealResultItem>>>>,
|
||||
delete_recovery_calls: Mutex<Vec<String>>,
|
||||
block_heal_object: Mutex<bool>,
|
||||
resume_disk: Mutex<Option<DiskStore>>,
|
||||
replacement_resume_disk: Mutex<Option<DiskStore>>,
|
||||
@@ -1544,6 +1546,7 @@ struct MockStorage {
|
||||
usage_baseline_error: Mutex<bool>,
|
||||
erasure_set_scopes: Mutex<Vec<(usize, usize)>>,
|
||||
disk_walk_calls: Mutex<Vec<String>>,
|
||||
disk_walk_items: Mutex<Option<Vec<HealListItem>>>,
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2323,6 +2326,34 @@ impl HealStorageAPI for MockStorage {
|
||||
Ok(HealResultItem::default())
|
||||
}
|
||||
|
||||
async fn recover_delete_transactions(
|
||||
&self,
|
||||
bucket: &str,
|
||||
_object: Option<&str>,
|
||||
_expected: Option<Uuid>,
|
||||
_opts: &HealOpts,
|
||||
) -> Result<Option<HealResultItem>> {
|
||||
self.delete_recovery_calls
|
||||
.lock()
|
||||
.expect("recovery calls")
|
||||
.push(bucket.to_owned());
|
||||
self.delete_recovery_results
|
||||
.lock()
|
||||
.expect("recovery results")
|
||||
.remove(bucket)
|
||||
.unwrap_or(Ok(None))
|
||||
}
|
||||
|
||||
async fn recover_delete_prefix_transactions(
|
||||
&self,
|
||||
bucket: &str,
|
||||
_prefix: &str,
|
||||
expected: Option<Uuid>,
|
||||
opts: &HealOpts,
|
||||
) -> Result<Option<HealResultItem>> {
|
||||
self.recover_delete_transactions(bucket, None, expected, opts).await
|
||||
}
|
||||
|
||||
async fn heal_format(&self, _dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
*self.global_format_calls.lock().unwrap() += 1;
|
||||
if let Some(error) = self.format_error.lock().unwrap().take() {
|
||||
@@ -2488,7 +2519,15 @@ impl HealStorageAPI for MockStorage {
|
||||
if continuation_token.is_some() {
|
||||
return Ok((Vec::new(), None, false));
|
||||
}
|
||||
Ok((vec![heal_item(&format!("{set_disk_id}-object"))], None, false))
|
||||
Ok((
|
||||
self.disk_walk_items
|
||||
.lock()
|
||||
.expect("disk-walk items")
|
||||
.clone()
|
||||
.unwrap_or_else(|| vec![heal_item(&format!("{set_disk_id}-object"))]),
|
||||
None,
|
||||
false,
|
||||
))
|
||||
}
|
||||
|
||||
async fn get_disk_for_resume(&self, _set_disk_id: &str) -> Result<DiskStore> {
|
||||
@@ -4432,6 +4471,336 @@ async fn erasure_set_bucket_prepass_failure_stops_before_object_heal() {
|
||||
assert!(storage.object_heal_opts.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_unresolved_delete_recovery_preserves_other_buckets_and_ordinary_heal() {
|
||||
use rustfs_madmin::heal_commands::DeleteRecoveryStatus;
|
||||
|
||||
for status in [
|
||||
DeleteRecoveryStatus {
|
||||
pending: 1,
|
||||
..Default::default()
|
||||
},
|
||||
DeleteRecoveryStatus {
|
||||
blocked: 1,
|
||||
..Default::default()
|
||||
},
|
||||
DeleteRecoveryStatus {
|
||||
failed: 1,
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
delete_recovery_results: Mutex::new(HashMap::from([(
|
||||
"bucket-a".to_owned(),
|
||||
Ok(Some(HealResultItem {
|
||||
bucket: "bucket-a".to_owned(),
|
||||
delete_recovery: Some(status),
|
||||
..Default::default()
|
||||
})),
|
||||
)])),
|
||||
resume_disk: Mutex::new(Some(disk)),
|
||||
list_each_bucket: true,
|
||||
..Default::default()
|
||||
});
|
||||
let buckets = vec!["bucket-a".to_owned(), "bucket-b".to_owned()];
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: buckets.clone(),
|
||||
set_disk_id: "pool_0_set_0".to_owned(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let error = task
|
||||
.heal_erasure_set(buckets.clone(), "pool_0_set_0".to_owned())
|
||||
.await
|
||||
.expect_err("unresolved recovery must not be reported as complete");
|
||||
|
||||
assert!(matches!(error, Error::TransientSkip { .. }), "recovery must remain retryable: {error}");
|
||||
assert_eq!(*storage.delete_recovery_calls.lock().expect("recovery calls"), buckets);
|
||||
assert_eq!(*storage.bucket_heal_calls.lock().expect("bucket calls"), buckets);
|
||||
assert_eq!(
|
||||
storage.object_heal_opts.lock().expect("object calls").len(),
|
||||
4,
|
||||
"ordinary object heal must run for both buckets"
|
||||
);
|
||||
assert!(
|
||||
task.get_result_items()
|
||||
.await
|
||||
.iter()
|
||||
.any(|item| item.bucket == "bucket-a" && item.delete_recovery == Some(status)),
|
||||
"the result retains the exact unresolved recovery category"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_delete_recovery_fatal_errors_stop_before_other_buckets() {
|
||||
for error in [
|
||||
Error::TaskCancelled,
|
||||
Error::TaskTimeout,
|
||||
Error::Storage(EcstoreError::PreconditionFailed),
|
||||
Error::Storage(EcstoreError::SlowDown),
|
||||
Error::Storage(EcstoreError::InvalidPath),
|
||||
] {
|
||||
let expected = error.to_string();
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let storage = Arc::new(MockStorage {
|
||||
delete_recovery_results: Mutex::new(HashMap::from([("bucket-a".to_owned(), Err(error))])),
|
||||
resume_disk: Mutex::new(Some(disk)),
|
||||
..Default::default()
|
||||
});
|
||||
let buckets = vec!["bucket-a".to_owned(), "bucket-b".to_owned()];
|
||||
let request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: buckets.clone(),
|
||||
set_disk_id: "pool_0_set_0".to_owned(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let actual = task
|
||||
.heal_erasure_set(buckets, "pool_0_set_0".to_owned())
|
||||
.await
|
||||
.expect_err("fence and control errors must terminate the prepass");
|
||||
|
||||
assert_eq!(actual.to_string(), expected);
|
||||
assert_eq!(*storage.delete_recovery_calls.lock().expect("recovery calls"), ["bucket-a"]);
|
||||
assert_eq!(*storage.bucket_heal_calls.lock().expect("bucket calls"), ["bucket-a"]);
|
||||
assert!(storage.object_heal_opts.lock().expect("object calls").is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_and_prefix_delete_recovery_stays_unresolved_after_ordinary_heal() {
|
||||
for prefix in [None, Some("logs/")] {
|
||||
let status = rustfs_madmin::heal_commands::DeleteRecoveryStatus {
|
||||
pending: 1,
|
||||
..Default::default()
|
||||
};
|
||||
let storage = Arc::new(MockStorage {
|
||||
delete_recovery_results: Mutex::new(HashMap::from([(
|
||||
"bucket-a".to_owned(),
|
||||
Ok(Some(HealResultItem {
|
||||
bucket: "bucket-a".to_owned(),
|
||||
delete_recovery: Some(status),
|
||||
..Default::default()
|
||||
})),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let heal_type = prefix.map_or_else(
|
||||
|| HealType::Bucket {
|
||||
bucket: "bucket-a".to_owned(),
|
||||
},
|
||||
|prefix| HealType::Prefix {
|
||||
bucket: "bucket-a".to_owned(),
|
||||
prefix: prefix.to_owned(),
|
||||
},
|
||||
);
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
heal_type,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
|
||||
let result = match prefix {
|
||||
Some(prefix) => task.heal_prefix("bucket-a", prefix).await,
|
||||
None => task.heal_bucket("bucket-a").await,
|
||||
};
|
||||
assert!(
|
||||
matches!(result, Err(Error::TransientSkip { .. })),
|
||||
"unfinished recovery must remain retryable: {result:?}"
|
||||
);
|
||||
assert_eq!(storage.object_heal_opts.lock().expect("object calls").len(), 2);
|
||||
assert!(
|
||||
task.get_result_items()
|
||||
.await
|
||||
.iter()
|
||||
.any(|item| item.delete_recovery == Some(status))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unresolved_delete_recovery_prevents_bucket_removal() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
delete_recovery_results: Mutex::new(HashMap::from([(
|
||||
"bucket-a".to_owned(),
|
||||
Ok(Some(HealResultItem {
|
||||
delete_recovery: Some(rustfs_madmin::heal_commands::DeleteRecoveryStatus {
|
||||
failed: 1,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_owned(),
|
||||
},
|
||||
HealOptions {
|
||||
remove_corrupted: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
|
||||
assert!(matches!(task.heal_bucket("bucket-a").await, Err(Error::TransientSkip { .. })));
|
||||
assert!(
|
||||
storage.bucket_heal_calls.lock().expect("bucket calls").is_empty(),
|
||||
"publication directories must remain until recovery resolves"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_erasure_set_delete_recovery_rescans_only_after_settlement() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(incarnation)),
|
||||
delete_recovery_results: Mutex::new(HashMap::from([(
|
||||
"bucket-a".to_owned(),
|
||||
Ok(Some(HealResultItem {
|
||||
delete_recovery: Some(rustfs_madmin::heal_commands::DeleteRecoveryStatus {
|
||||
blocked: 1,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})),
|
||||
)])),
|
||||
resume_disk: Mutex::new(Some(disk.clone())),
|
||||
disk_walk_items: Mutex::new(Some(vec![heal_item("object-a"), heal_item("object-b")])),
|
||||
..Default::default()
|
||||
});
|
||||
let buckets = vec!["bucket-a".to_owned(), "bucket-b".to_owned()];
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: buckets.clone(),
|
||||
set_disk_id: "pool_0_set_0".to_owned(),
|
||||
},
|
||||
HealOptions {
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let error = task
|
||||
.heal_erasure_set(buckets.clone(), "pool_0_set_0".to_owned())
|
||||
.await
|
||||
.expect_err("administrator recovery cannot complete a set with unresolved recovery");
|
||||
assert!(matches!(error, Error::TransientSkip { .. }), "typed unresolved recovery: {error}");
|
||||
assert_eq!(storage.object_heal_opts.lock().expect("object calls").len(), 4);
|
||||
let checkpoint = CheckpointManager::load_from_disk(disk.clone(), &task.id)
|
||||
.await
|
||||
.expect("retained administrator checkpoint")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
assert_eq!(
|
||||
checkpoint.current_bucket_index, 2,
|
||||
"ordinary work advances independently from hidden recovery"
|
||||
);
|
||||
assert!(!checkpoint.admin.expect("administrator outcome").completed);
|
||||
assert_eq!(*storage.delete_recovery_calls.lock().expect("recovery calls"), buckets);
|
||||
storage.delete_recovery_results.lock().expect("recovery results").insert(
|
||||
"bucket-a".to_owned(),
|
||||
Ok(Some(HealResultItem {
|
||||
delete_recovery: Some(rustfs_madmin::heal_commands::DeleteRecoveryStatus {
|
||||
failed: 1,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
})),
|
||||
);
|
||||
let error = task
|
||||
.heal_erasure_set(buckets.clone(), "pool_0_set_0".to_owned())
|
||||
.await
|
||||
.expect_err("completed ordinary buckets must still retry their hidden recovery");
|
||||
assert!(matches!(error, Error::TransientSkip { .. }));
|
||||
assert_eq!(
|
||||
storage.object_heal_opts.lock().expect("object calls").len(),
|
||||
4,
|
||||
"retry must not replay payload healing"
|
||||
);
|
||||
|
||||
*storage.bucket_incarnation_id.lock().expect("bucket identity") = Some(Uuid::new_v4());
|
||||
let error = task
|
||||
.heal_erasure_set(buckets.clone(), "pool_0_set_0".to_owned())
|
||||
.await
|
||||
.expect_err("recovery-only retries must retain the admitted bucket generation");
|
||||
assert!(matches!(error, Error::StaleBucketIncarnation { .. }));
|
||||
*storage.bucket_incarnation_id.lock().expect("bucket identity") = Some(incarnation);
|
||||
|
||||
// A background worker may restore and collect the hidden transaction
|
||||
// before this retry, so the recovery result can now be None.
|
||||
*storage.disk_walk_items.lock().expect("disk-walk items") =
|
||||
Some(vec![heal_item("object-a"), heal_item("object-b"), heal_item("restored")]);
|
||||
let error = task
|
||||
.heal_erasure_set(buckets.clone(), "pool_0_set_0".to_owned())
|
||||
.await
|
||||
.expect_err("settlement must invalidate overlapping ordinary coverage");
|
||||
assert!(matches!(error, Error::TransientSkip { .. }));
|
||||
assert_eq!(storage.object_heal_opts.lock().expect("object calls").len(), 4);
|
||||
task.heal_erasure_set(buckets, "pool_0_set_0".to_owned())
|
||||
.await
|
||||
.expect("a fresh coverage pass can complete");
|
||||
assert_eq!(storage.object_heal_opts.lock().expect("object calls").len(), 10);
|
||||
assert_eq!(
|
||||
storage
|
||||
.healed_objects
|
||||
.lock()
|
||||
.expect("healed objects")
|
||||
.iter()
|
||||
.filter(|object| object.as_str() == "restored")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
*storage.delete_recovery_calls.lock().expect("recovery calls"),
|
||||
[
|
||||
"bucket-a", "bucket-b", "bucket-a", "bucket-b", "bucket-a", "bucket-b", "bucket-a", "bucket-b"
|
||||
]
|
||||
);
|
||||
let checkpoint = CheckpointManager::load_from_disk(disk, &task.id)
|
||||
.await
|
||||
.expect("completed administrator receipt")
|
||||
.get_checkpoint()
|
||||
.await;
|
||||
let admin = checkpoint.admin.expect("administrator outcome");
|
||||
assert!(admin.completed);
|
||||
assert_eq!(admin.outcome.counters.processed, 6, "the fresh coverage pass owns its counters");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_set_heal_applies_usage_baseline_to_progress() {
|
||||
let temp = TempDir::new().expect("temporary directory should be created");
|
||||
|
||||
@@ -702,6 +702,180 @@ mod serial_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn historical_rebalance_topology_allows_format_and_object_rebuild() {
|
||||
const STACK_SIZE: usize = 8 * 1024 * 1024;
|
||||
std::thread::Builder::new()
|
||||
.name("historical-rebalance-rebuild".to_owned())
|
||||
.stack_size(STACK_SIZE)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(4)
|
||||
.thread_stack_size(STACK_SIZE)
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("historical rebalance recovery runtime");
|
||||
runtime.block_on(historical_rebalance_topology_allows_format_and_object_rebuild_case());
|
||||
})
|
||||
.expect("historical rebalance recovery test thread")
|
||||
.join()
|
||||
.expect("historical rebalance recovery test result");
|
||||
}
|
||||
|
||||
async fn historical_rebalance_topology_allows_format_and_object_rebuild_case() {
|
||||
use storage_api::integration::{
|
||||
DiskAPI as _, DiskError, DiskOption, RUSTFS_META_BUCKET, RebalStatus, RebalanceMeta, RebalanceStats, new_disk,
|
||||
};
|
||||
|
||||
let temp = tempfile::tempdir().expect("isolated historical rebalance fixture");
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp.path())
|
||||
.build()
|
||||
.await;
|
||||
let ecstore = env.ecstore;
|
||||
let disks = env.disk_paths;
|
||||
let storage = Arc::new(ECStoreHealStorage::new(ecstore.clone()));
|
||||
assert_eq!(ecstore.pools.len(), 1);
|
||||
let bucket = "historical-rebalance-rebuild";
|
||||
let object = "non-inline-object";
|
||||
let expected = non_inline_test_data();
|
||||
create_test_bucket(&ecstore, bucket).await;
|
||||
upload_test_object(&ecstore, bucket, object, &expected).await;
|
||||
wait_for_object_copies(&disks, bucket, object).await;
|
||||
|
||||
// Model the persisted post-removal state. This test does not execute a
|
||||
// rebalance worker, decommission lifecycle, or mount-identity handoff.
|
||||
let mut historical = RebalanceMeta {
|
||||
id: uuid::Uuid::new_v4().to_string(),
|
||||
pool_stats: vec![RebalanceStats::default(), RebalanceStats::default()],
|
||||
..Default::default()
|
||||
};
|
||||
for pool in &mut historical.pool_stats {
|
||||
pool.participating = true;
|
||||
pool.info.status = RebalStatus::Completed;
|
||||
}
|
||||
historical
|
||||
.save_with_opts(
|
||||
ecstore.pools[0].clone(),
|
||||
ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("persist completed historical two-pool metadata on the current metadata pool");
|
||||
let historical_bytes = ecstore.pools[0]
|
||||
.get_object_reader(RUSTFS_META_BUCKET, "rebalance.bin", None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("open persisted historical metadata")
|
||||
.read_all()
|
||||
.await
|
||||
.expect("read persisted historical metadata");
|
||||
let old_format: serde_json::Value = serde_json::from_slice(
|
||||
&fs::read(disks[0].join(RUSTFS_META_BUCKET).join("format.json"))
|
||||
.await
|
||||
.expect("read target format"),
|
||||
)
|
||||
.expect("decode target format");
|
||||
let target_object = disks[0].join(bucket).join(object);
|
||||
let old_shard = fs::read(find_part_file(&target_object).expect("original target shard"))
|
||||
.await
|
||||
.expect("read original target shard bytes");
|
||||
|
||||
// Preserve the root inode held by disk I/O, while replacing its contents
|
||||
// with an empty target, as in the existing fresh-target recovery fixture.
|
||||
for entry in std::fs::read_dir(&disks[0]).expect("read replacement target root") {
|
||||
let entry = entry.expect("replacement target entry");
|
||||
if entry.file_type().expect("replacement entry type").is_dir() {
|
||||
fs::remove_dir_all(entry.path()).await.expect("empty target directory");
|
||||
} else {
|
||||
fs::remove_file(entry.path()).await.expect("empty target file");
|
||||
}
|
||||
}
|
||||
assert!(!disks[0].join(RUSTFS_META_BUCKET).join("format.json").exists());
|
||||
assert!(!target_object.exists());
|
||||
// A replacement reconnects with a fresh handle. Reusing the old handle
|
||||
// would retain its cached format bytes despite the emptied directory.
|
||||
let endpoint = ecstore.pools[0].endpoints.endpoints.as_ref()[0].clone();
|
||||
let replacement = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("open a fresh replacement disk handle");
|
||||
assert!(matches!(replacement.get_disk_id().await, Err(DiskError::UnformattedDisk)));
|
||||
ecstore.pools[0].get_disks(0).disks.write().await[0] = Some(replacement);
|
||||
let target = endpoint.to_string();
|
||||
let (_, format_error) = storage
|
||||
.heal_replacement_format(false, 0, 0, std::slice::from_ref(&target))
|
||||
.await
|
||||
.expect("historical topology must admit target-scoped format repair");
|
||||
assert!(format_error.is_none(), "target format repair: {format_error:?}");
|
||||
let repaired_format: serde_json::Value = serde_json::from_slice(
|
||||
&fs::read(disks[0].join(RUSTFS_META_BUCKET).join("format.json"))
|
||||
.await
|
||||
.expect("read repaired target format"),
|
||||
)
|
||||
.expect("decode repaired target format");
|
||||
assert_eq!(
|
||||
repaired_format["xl"]["this"], old_format["xl"]["this"],
|
||||
"format repair must retain the original slot UUID"
|
||||
);
|
||||
assert!(
|
||||
!target_object.exists(),
|
||||
"format repair alone must not be mistaken for object reconstruction"
|
||||
);
|
||||
|
||||
let mut request = HealRequest::new(
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![bucket.to_owned()],
|
||||
set_disk_id: "pool_0_set_0".to_owned(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let task = HealTask::from_request(request, storage);
|
||||
task.execute()
|
||||
.await
|
||||
.expect("rebuild objects after historical-topology format repair");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
// Verify target-side physical repair before a GET could perform read repair.
|
||||
assert!(target_object.join("xl.meta").exists());
|
||||
let rebuilt_shard = fs::read(find_part_file(&target_object).expect("rebuilt target shard"))
|
||||
.await
|
||||
.expect("read rebuilt target shard bytes");
|
||||
assert_eq!(rebuilt_shard, old_shard, "the replacement shard must contain the original encoded bytes");
|
||||
let actual = ecstore
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("open rebuilt object")
|
||||
.read_all()
|
||||
.await
|
||||
.expect("read rebuilt object bytes");
|
||||
assert_eq!(actual, expected, "reconstructed object bytes must be unchanged");
|
||||
let retained_history = ecstore.pools[0]
|
||||
.get_object_reader(RUSTFS_META_BUCKET, "rebalance.bin", None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("open retained historical metadata")
|
||||
.read_all()
|
||||
.await
|
||||
.expect("read retained historical metadata");
|
||||
assert_eq!(retained_history, historical_bytes, "healing must retain the historical rebalance payload");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn replacement_delete_recovery_on_fresh_target() {
|
||||
|
||||
@@ -28,6 +28,7 @@ pub(crate) mod integration {
|
||||
pub(crate) use rustfs_ecstore::api::object::{
|
||||
ObjectOptions, PutObjReader, ShardIntegrityWriteMode, WriteCommitGuard, WriteCompletion,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::rebalance::{RebalStatus, RebalanceMeta, RebalanceStats};
|
||||
pub(crate) use rustfs_ecstore::api::storage::ECStore;
|
||||
pub(crate) use rustfs_storage_api::BucketOperations;
|
||||
pub(crate) use rustfs_storage_api::MakeBucketOptions;
|
||||
|
||||
@@ -30,6 +30,7 @@ harness = false
|
||||
|
||||
[features]
|
||||
default = []
|
||||
diagnostic-read-version-caller = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::s3_http_metrics::S3HttpOperationContext;
|
||||
use metrics::{counter, gauge};
|
||||
#[cfg(not(test))]
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{
|
||||
Arc, LazyLock, OnceLock, RwLock,
|
||||
@@ -65,6 +68,340 @@ pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE: &str = "batch_read
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_COALESCER_WAIT: &str = "batch_read_version_coalescer_wait";
|
||||
pub const INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_MAP: &str = "batch_read_version_response_map";
|
||||
|
||||
const INTERNODE_READ_VERSION_REQUESTS_BY_CALLER_TOTAL: &str =
|
||||
"rustfs_system_network_internode_read_version_requests_by_caller_total";
|
||||
|
||||
/// An attribution dimension is absent, common to all items, or heterogeneous.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ReadVersionAttribution<T> {
|
||||
Absent,
|
||||
Common(T),
|
||||
Mixed,
|
||||
}
|
||||
|
||||
/// Storage entry names describe call paths, not proof of an external request.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ReadVersionEntrySource {
|
||||
Unclassified,
|
||||
ConfigRead,
|
||||
ConfigWrite,
|
||||
ScannerConfigWrite,
|
||||
WritePreconditionRead,
|
||||
ObjectInfoRead,
|
||||
AppPutOwner,
|
||||
PutCommitOwner,
|
||||
LifecycleRecovery,
|
||||
RecoveryDispositionMaintenance,
|
||||
StoreObjectReader,
|
||||
DataUsageCacheRead,
|
||||
ServerInfoPools,
|
||||
ScannerBacklogRead,
|
||||
ScannerRevisionRead,
|
||||
ScannerCycleStateRead,
|
||||
TierConfigReload,
|
||||
ObjectRead,
|
||||
ListObjects,
|
||||
MetadataRead,
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
impl ReadVersionEntrySource {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unclassified => "unclassified",
|
||||
Self::ConfigRead => "config_read",
|
||||
Self::ConfigWrite => "config_write",
|
||||
Self::ScannerConfigWrite => "scanner_config_write",
|
||||
Self::WritePreconditionRead => "write_precondition_read",
|
||||
Self::ObjectInfoRead => "object_info_read",
|
||||
Self::AppPutOwner => "app_put_owner",
|
||||
Self::PutCommitOwner => "put_commit_owner",
|
||||
Self::LifecycleRecovery => "lifecycle_recovery",
|
||||
Self::RecoveryDispositionMaintenance => "recovery_disposition_maintenance",
|
||||
Self::StoreObjectReader => "store_object_reader",
|
||||
Self::DataUsageCacheRead => "data_usage_cache_read",
|
||||
Self::ServerInfoPools => "server_info_pools",
|
||||
Self::ScannerBacklogRead => "scanner_backlog_read",
|
||||
Self::ScannerRevisionRead => "scanner_revision_read",
|
||||
Self::ScannerCycleStateRead => "scanner_cycle_state_read",
|
||||
Self::TierConfigReload => "tier_config_reload",
|
||||
Self::ObjectRead => "object_read",
|
||||
Self::ListObjects => "list_objects",
|
||||
Self::MetadataRead => "metadata_read",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum ReadVersionContextBoundary {
|
||||
Entry,
|
||||
FullWait,
|
||||
EarlyStop,
|
||||
ListProducer,
|
||||
ListGather,
|
||||
CoalescerEnqueue,
|
||||
UnaryFallback,
|
||||
EntryToRpc,
|
||||
RemoteRpc,
|
||||
ScannerCacheLoad,
|
||||
ScannerCycleStateLoad,
|
||||
ConfigWrite,
|
||||
ScannerConfigWrite,
|
||||
WritePreconditionLookup,
|
||||
ObjectInfoRead,
|
||||
AppPutOwner,
|
||||
PutCommitOwner,
|
||||
TierFreeVersionSourceLookup,
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
impl ReadVersionContextBoundary {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Entry => "entry",
|
||||
Self::FullWait => "full_wait",
|
||||
Self::EarlyStop => "early_stop",
|
||||
Self::ListProducer => "list_producer",
|
||||
Self::ListGather => "list_gather",
|
||||
Self::CoalescerEnqueue => "coalescer_enqueue",
|
||||
Self::UnaryFallback => "unary_fallback",
|
||||
Self::EntryToRpc => "entry_to_rpc",
|
||||
Self::RemoteRpc => "remote_rpc",
|
||||
Self::ScannerCacheLoad => "scanner_cache_load",
|
||||
Self::ScannerCycleStateLoad => "scanner_cycle_state_load",
|
||||
Self::ConfigWrite => "config_write",
|
||||
Self::ScannerConfigWrite => "scanner_config_write",
|
||||
Self::WritePreconditionLookup => "write_precondition_lookup",
|
||||
Self::ObjectInfoRead => "object_info_read",
|
||||
Self::AppPutOwner => "app_put_owner",
|
||||
Self::PutCommitOwner => "put_commit_owner",
|
||||
Self::TierFreeVersionSourceLookup => "tier_free_version_source_lookup",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
fn context_transition<T: Eq>(parent: Option<T>, child: Option<T>) -> &'static str {
|
||||
match (parent, child) {
|
||||
(None, None) => "absent",
|
||||
(Some(_), None) => "lost",
|
||||
(None, Some(_)) => "gained",
|
||||
(Some(parent), Some(child)) if parent == child => "preserved",
|
||||
(Some(_), Some(_)) => "changed",
|
||||
}
|
||||
}
|
||||
|
||||
/// Count observations separately from RPCs, using only bounded context labels.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub fn record_read_version_context_transition(
|
||||
source: ReadVersionEntrySource,
|
||||
boundary: ReadVersionContextBoundary,
|
||||
parent_api: Option<S3HttpOperationContext>,
|
||||
child_api: Option<S3HttpOperationContext>,
|
||||
parent_caller: Option<ReadVersionRequestCaller>,
|
||||
child_caller: Option<ReadVersionRequestCaller>,
|
||||
mode: Option<ReadVersionRequestMode>,
|
||||
) {
|
||||
if !crate::get_stage_metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
counter!(
|
||||
"rustfs_storage_diagnostic_context_transitions_total",
|
||||
"entry_source" => source.as_str(),
|
||||
"boundary" => boundary.as_str(),
|
||||
"parent_api_operation" => parent_api.map(S3HttpOperationContext::operation_name).unwrap_or("unattributed"),
|
||||
"child_api_operation" => child_api.map(S3HttpOperationContext::operation_name).unwrap_or("unattributed"),
|
||||
"parent_caller" => parent_caller.map(ReadVersionRequestCaller::as_str).unwrap_or("unattributed"),
|
||||
"child_caller" => child_caller.map(ReadVersionRequestCaller::as_str).unwrap_or("unattributed"),
|
||||
"mode" => mode.map(ReadVersionRequestMode::as_str).unwrap_or("not_applicable"),
|
||||
"api_context" => if source == ReadVersionEntrySource::Unclassified && matches!(boundary, ReadVersionContextBoundary::RemoteRpc) {
|
||||
"unobserved"
|
||||
} else { context_transition(parent_api, child_api) },
|
||||
"caller_context" => if source == ReadVersionEntrySource::Unclassified && matches!(boundary, ReadVersionContextBoundary::RemoteRpc) {
|
||||
"unobserved"
|
||||
} else { context_transition(parent_caller, child_caller) },
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Count a native batch RPC once; item observations and unary fallback are separate.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub fn record_batch_read_version_request_by_caller(
|
||||
operation: ReadVersionAttribution<S3HttpOperationContext>,
|
||||
caller: ReadVersionAttribution<ReadVersionRequestCaller>,
|
||||
source: ReadVersionAttribution<ReadVersionEntrySource>,
|
||||
mode: ReadVersionRequestMode,
|
||||
) {
|
||||
let api_label = match operation {
|
||||
ReadVersionAttribution::Absent => "unattributed",
|
||||
ReadVersionAttribution::Common(operation) => operation.operation_name(),
|
||||
ReadVersionAttribution::Mixed => "mixed",
|
||||
};
|
||||
let caller_label = match caller {
|
||||
ReadVersionAttribution::Absent => "unattributed",
|
||||
ReadVersionAttribution::Common(caller) => caller.as_str(),
|
||||
ReadVersionAttribution::Mixed => "mixed",
|
||||
};
|
||||
let source_label = match source {
|
||||
ReadVersionAttribution::Absent => "unclassified",
|
||||
ReadVersionAttribution::Common(source) => source.as_str(),
|
||||
ReadVersionAttribution::Mixed => "mixed",
|
||||
};
|
||||
counter!(
|
||||
"rustfs_system_network_internode_batch_read_version_requests_by_caller_total",
|
||||
"api_operation" => api_label,
|
||||
"caller" => caller_label,
|
||||
"entry_source" => source_label,
|
||||
"mode" => mode.as_str(),
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
const READ_VERSION_REQUEST_MODE_COUNT: usize = 5;
|
||||
const READ_VERSION_REQUEST_CALLER_COUNT: usize = 18;
|
||||
|
||||
#[cfg(not(test))]
|
||||
static READ_VERSION_CALLER_COUNTERS: LazyLock<Box<[OnceLock<metrics::Counter>]>> = LazyLock::new(|| {
|
||||
std::iter::repeat_with(OnceLock::new)
|
||||
.take((S3Operation::ALL.len() + 1) * READ_VERSION_REQUEST_MODE_COUNT * READ_VERSION_REQUEST_CALLER_COUNT)
|
||||
.collect::<Vec<_>>()
|
||||
.into_boxed_slice()
|
||||
});
|
||||
|
||||
/// Coarse read mode for diagnostic attribution of remote metadata reads.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum ReadVersionRequestMode {
|
||||
MetadataOnly,
|
||||
Data,
|
||||
Healing,
|
||||
IncludeFreeVersions,
|
||||
Combined,
|
||||
}
|
||||
|
||||
/// Bounded internal call-site class for an outgoing ReadVersion RPC.
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
#[repr(usize)]
|
||||
pub enum ReadVersionRequestCaller {
|
||||
#[default]
|
||||
Unattributed,
|
||||
Peer,
|
||||
Startup,
|
||||
Heal,
|
||||
ListObjects,
|
||||
Object,
|
||||
MetadataRead,
|
||||
Multipart,
|
||||
DiskStore,
|
||||
DiskDispatch,
|
||||
SetDiskRead,
|
||||
SetDiskIo,
|
||||
Rebalance,
|
||||
Decommission,
|
||||
Lifecycle,
|
||||
TierRecovery,
|
||||
CacheValue,
|
||||
Other,
|
||||
}
|
||||
|
||||
impl ReadVersionRequestCaller {
|
||||
pub const ALL: [Self; READ_VERSION_REQUEST_CALLER_COUNT] = [
|
||||
Self::Unattributed,
|
||||
Self::Peer,
|
||||
Self::Startup,
|
||||
Self::Heal,
|
||||
Self::ListObjects,
|
||||
Self::Object,
|
||||
Self::MetadataRead,
|
||||
Self::Multipart,
|
||||
Self::DiskStore,
|
||||
Self::DiskDispatch,
|
||||
Self::SetDiskRead,
|
||||
Self::SetDiskIo,
|
||||
Self::Rebalance,
|
||||
Self::Decommission,
|
||||
Self::Lifecycle,
|
||||
Self::TierRecovery,
|
||||
Self::CacheValue,
|
||||
Self::Other,
|
||||
];
|
||||
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unattributed => "unattributed",
|
||||
Self::Peer => "peer",
|
||||
Self::Startup => "startup",
|
||||
Self::Heal => "heal",
|
||||
Self::ListObjects => "list_objects",
|
||||
Self::Object => "object",
|
||||
Self::MetadataRead => "metadata_read",
|
||||
Self::Multipart => "multipart",
|
||||
Self::DiskStore => "disk_store",
|
||||
Self::DiskDispatch => "disk_dispatch",
|
||||
Self::SetDiskRead => "set_disk_read",
|
||||
Self::SetDiskIo => "set_disk_io",
|
||||
Self::Rebalance => "rebalance",
|
||||
Self::Decommission => "decommission",
|
||||
Self::Lifecycle => "lifecycle",
|
||||
Self::TierRecovery => "tier_recovery",
|
||||
Self::CacheValue => "cache_value",
|
||||
Self::Other => "other",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReadVersionRequestMode {
|
||||
const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::MetadataOnly => "metadata_only",
|
||||
Self::Data => "data",
|
||||
Self::Healing => "healing",
|
||||
Self::IncludeFreeVersions => "include_free_versions",
|
||||
Self::Combined => "combined",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a bounded diagnostic caller class for one outgoing ReadVersion RPC.
|
||||
/// The caller label is either a fixed S3 operation name or `unattributed`.
|
||||
pub fn record_read_version_request_by_caller(
|
||||
operation: Option<S3HttpOperationContext>,
|
||||
caller: ReadVersionRequestCaller,
|
||||
mode: ReadVersionRequestMode,
|
||||
) {
|
||||
let api_operation = operation
|
||||
.map(S3HttpOperationContext::operation_name)
|
||||
.unwrap_or("unattributed");
|
||||
|
||||
#[cfg(test)]
|
||||
counter!(
|
||||
INTERNODE_READ_VERSION_REQUESTS_BY_CALLER_TOTAL,
|
||||
"api_operation" => api_operation,
|
||||
"caller" => caller.as_str(),
|
||||
"mode" => mode.as_str(),
|
||||
)
|
||||
.increment(1);
|
||||
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
let caller_index = operation.map_or(S3Operation::ALL.len(), |context| context.operation_index());
|
||||
let counter_index = (caller_index * READ_VERSION_REQUEST_MODE_COUNT + mode as usize) * READ_VERSION_REQUEST_CALLER_COUNT
|
||||
+ caller as usize;
|
||||
READ_VERSION_CALLER_COUNTERS[counter_index]
|
||||
.get_or_init(|| {
|
||||
counter!(
|
||||
INTERNODE_READ_VERSION_REQUESTS_BY_CALLER_TOTAL,
|
||||
"api_operation" => api_operation,
|
||||
"caller" => caller.as_str(),
|
||||
"mode" => mode.as_str(),
|
||||
)
|
||||
})
|
||||
.increment(1);
|
||||
}
|
||||
}
|
||||
|
||||
const OPERATION_LABEL: &str = "operation";
|
||||
const BACKEND_LABEL: &str = "backend";
|
||||
const SERVER_LABEL: &str = "server";
|
||||
@@ -1231,6 +1568,61 @@ mod tests {
|
||||
assert_eq!(snapshot.replay_cache_evictions_total, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_version_caller_metric_uses_only_bounded_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
with_local_recorder(&recorder, || {
|
||||
let mut request = crate::s3_http_metrics::S3HttpRequestGuard::new("GET");
|
||||
let operation = request
|
||||
.in_scope(|| {
|
||||
crate::s3_http_metrics::observe_s3_http_operation(rustfs_s3_ops::S3Operation::GetObject);
|
||||
crate::s3_http_metrics::current_s3_http_operation_context()
|
||||
})
|
||||
.expect("S3 request should expose its dispatched operation");
|
||||
record_read_version_request_by_caller(
|
||||
Some(operation),
|
||||
ReadVersionRequestCaller::SetDiskRead,
|
||||
ReadVersionRequestMode::Data,
|
||||
);
|
||||
record_read_version_request_by_caller(None, ReadVersionRequestCaller::Peer, ReadVersionRequestMode::MetadataOnly);
|
||||
});
|
||||
|
||||
let rows = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_READ_VERSION_REQUESTS_BY_CALLER_TOTAL)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(rows.len(), 2);
|
||||
let labels = rows
|
||||
.iter()
|
||||
.map(|(composite, _, _, _)| {
|
||||
composite
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect::<HashMap<_, _>>()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert!(labels.iter().any(|labels| {
|
||||
labels.get("api_operation").map(String::as_str) == Some("s3:GetObject")
|
||||
&& labels.get("caller").map(String::as_str) == Some("set_disk_read")
|
||||
&& labels.get("mode").map(String::as_str) == Some("data")
|
||||
}));
|
||||
assert!(labels.iter().any(|labels| {
|
||||
labels.get("api_operation").map(String::as_str) == Some("unattributed")
|
||||
&& labels.get("mode").map(String::as_str) == Some("metadata_only")
|
||||
&& labels.get("caller").map(String::as_str) == Some("peer")
|
||||
}));
|
||||
assert!(labels.iter().all(|labels| {
|
||||
labels.len() == 3
|
||||
&& labels
|
||||
.keys()
|
||||
.all(|key| key == "api_operation" || key == "caller" || key == "mode")
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn operation_stage_duration_records_low_cardinality_stage_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
|
||||
@@ -546,8 +546,8 @@ pub use io_metrics::{
|
||||
pub use list_objects_metrics::{
|
||||
LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED, LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED,
|
||||
LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR, LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_OK, LIST_OBJECTS_SOURCE_WALKER,
|
||||
ListObjectsGatherObservation, ListObjectsIndexPageObservation, ListObjectsLocalReadDirObservation, init_list_objects_metrics,
|
||||
record_list_objects_gather, record_list_objects_index_attempt, record_list_objects_index_fallback,
|
||||
ListObjectsGatherCaller, ListObjectsGatherObservation, ListObjectsIndexPageObservation, ListObjectsLocalReadDirObservation,
|
||||
init_list_objects_metrics, record_list_objects_gather, record_list_objects_index_attempt, record_list_objects_index_fallback,
|
||||
record_list_objects_index_live_verify_failure, record_list_objects_index_served, record_list_objects_local_read_dir,
|
||||
record_list_objects_merge,
|
||||
};
|
||||
|
||||
@@ -22,7 +22,31 @@ pub const LIST_OBJECTS_GATHER_OUTCOME_LIMIT_REACHED: &str = "limit_reached";
|
||||
pub const LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED: &str = "input_closed";
|
||||
pub const LIST_OBJECTS_MERGE_OUTCOME_STARTED: &str = "started";
|
||||
|
||||
/// Bounded caller class for a ListObjects walker gather.
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub enum ListObjectsGatherCaller {
|
||||
#[default]
|
||||
Unattributed,
|
||||
S3List,
|
||||
S3ListVersions,
|
||||
Lifecycle,
|
||||
Other,
|
||||
}
|
||||
|
||||
impl ListObjectsGatherCaller {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Unattributed => "unattributed",
|
||||
Self::S3List => "s3_list",
|
||||
Self::S3ListVersions => "s3_list_versions",
|
||||
Self::Lifecycle => "lifecycle",
|
||||
Self::Other => "other",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const LIST_OBJECTS_GATHER_TOTAL: &str = "rustfs_s3_list_objects_gather_total";
|
||||
const LIST_OBJECTS_GATHER_CALLER_TOTAL: &str = "rustfs_s3_list_objects_gather_caller_total";
|
||||
const LIST_OBJECTS_GATHER_DURATION_MS: &str = "rustfs_s3_list_objects_gather_duration_ms";
|
||||
const LIST_OBJECTS_GATHER_SCANNED_ENTRIES: &str = "rustfs_s3_list_objects_gather_scanned_entries";
|
||||
const LIST_OBJECTS_GATHER_RETURNED_ENTRIES: &str = "rustfs_s3_list_objects_gather_returned_entries";
|
||||
@@ -53,6 +77,7 @@ pub const LIST_OBJECTS_LOCAL_READ_DIR_OUTCOME_ERROR: &str = "error";
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct ListObjectsGatherObservation {
|
||||
pub source: &'static str,
|
||||
pub caller: ListObjectsGatherCaller,
|
||||
pub outcome: &'static str,
|
||||
pub limit: i32,
|
||||
pub scanned_entries: usize,
|
||||
@@ -213,6 +238,7 @@ pub fn record_list_objects_gather(observation: ListObjectsGatherObservation) {
|
||||
} else {
|
||||
count_as_f64(observation.scanned_entries) / count_as_f64(observation.returned_entries)
|
||||
};
|
||||
let api_operation = crate::s3_http_metrics::current_s3_http_operation_name().unwrap_or("unattributed");
|
||||
|
||||
counter!(
|
||||
LIST_OBJECTS_GATHER_TOTAL,
|
||||
@@ -223,6 +249,14 @@ pub fn record_list_objects_gather(observation: ListObjectsGatherObservation) {
|
||||
"has_marker" => bool_label(observation.has_marker),
|
||||
)
|
||||
.increment(1);
|
||||
counter!(
|
||||
LIST_OBJECTS_GATHER_CALLER_TOTAL,
|
||||
"api_operation" => api_operation,
|
||||
"caller" => observation.caller.as_str(),
|
||||
"source" => observation.source,
|
||||
"outcome" => observation.outcome,
|
||||
)
|
||||
.increment(1);
|
||||
histogram!(
|
||||
LIST_OBJECTS_GATHER_DURATION_MS,
|
||||
"source" => observation.source,
|
||||
@@ -432,6 +466,7 @@ mod tests {
|
||||
let rows = recorded(|| {
|
||||
record_list_objects_gather(ListObjectsGatherObservation {
|
||||
source: LIST_OBJECTS_SOURCE_WALKER,
|
||||
caller: ListObjectsGatherCaller::Unattributed,
|
||||
outcome: LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED,
|
||||
limit: 1001,
|
||||
scanned_entries: 42,
|
||||
@@ -444,6 +479,17 @@ mod tests {
|
||||
});
|
||||
|
||||
assert_eq!(counter_total(&rows, LIST_OBJECTS_GATHER_TOTAL), Some(1));
|
||||
assert_eq!(counter_total(&rows, LIST_OBJECTS_GATHER_CALLER_TOTAL), Some(1));
|
||||
let caller_labels = rows
|
||||
.iter()
|
||||
.find(|(composite, _, _, _)| composite.key().name() == LIST_OBJECTS_GATHER_CALLER_TOTAL)
|
||||
.expect("internal caller counter should be emitted")
|
||||
.0
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
assert_eq!(caller_labels.get("api_operation").map(String::as_str), Some("unattributed"));
|
||||
// Zero returned entries must not divide: the amplification reports the
|
||||
// scan count itself rather than an infinity or a NaN.
|
||||
assert_eq!(histogram_samples(&rows, LIST_OBJECTS_GATHER_SCAN_AMPLIFICATION), vec![42.0]);
|
||||
@@ -452,6 +498,44 @@ mod tests {
|
||||
assert_eq!(histogram_samples(&rows, LIST_OBJECTS_GATHER_DURATION_MS), vec![3.5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gather_caller_metric_attributes_the_s3_api_operation() {
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
|
||||
let rows = recorded(|| {
|
||||
let mut request = crate::s3_http_metrics::S3HttpRequestGuard::new("GET");
|
||||
request.in_scope(|| {
|
||||
crate::s3_http_metrics::observe_s3_http_operation(S3Operation::ListObjectsV2);
|
||||
record_list_objects_gather(ListObjectsGatherObservation {
|
||||
source: LIST_OBJECTS_SOURCE_WALKER,
|
||||
caller: ListObjectsGatherCaller::S3List,
|
||||
outcome: LIST_OBJECTS_GATHER_OUTCOME_INPUT_CLOSED,
|
||||
limit: 1000,
|
||||
scanned_entries: 1,
|
||||
returned_entries: 1,
|
||||
duration_ms: 1.0,
|
||||
has_prefix: false,
|
||||
has_delimiter: false,
|
||||
has_marker: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
assert_eq!(counter_total(&rows, LIST_OBJECTS_GATHER_CALLER_TOTAL), Some(1));
|
||||
let labels = rows
|
||||
.iter()
|
||||
.find(|(composite, _, _, _)| composite.key().name() == LIST_OBJECTS_GATHER_CALLER_TOTAL)
|
||||
.expect("caller counter should be emitted")
|
||||
.0
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect::<std::collections::HashMap<_, _>>();
|
||||
assert_eq!(labels.get("api_operation").map(String::as_str), Some("s3:ListObjectsV2"));
|
||||
assert_eq!(labels.get("caller").map(String::as_str), Some("s3_list"));
|
||||
assert_eq!(labels.get("source").map(String::as_str), Some(LIST_OBJECTS_SOURCE_WALKER));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_records_a_zero_read_quorum_rather_than_skipping_it() {
|
||||
let rows = recorded(|| record_list_objects_merge(LIST_OBJECTS_SOURCE_WALKER, 4, 0));
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use std::cell::Cell;
|
||||
use std::future::Future;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{LazyLock, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -38,6 +39,81 @@ const UNKNOWN_OPERATION: usize = S3Operation::ALL.len();
|
||||
static COUNTERS: LazyLock<HttpOutcomeCounters> = LazyLock::new(HttpOutcomeCounters::new);
|
||||
static COMPLETION_OBSERVER: OnceLock<(S3HttpCompletionObserverEnabled, S3HttpCompletionObserver)> = OnceLock::new();
|
||||
|
||||
/// A bounded S3 operation captured from the current external-request context.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct S3HttpOperationContext(S3Operation);
|
||||
|
||||
impl S3HttpOperationContext {
|
||||
/// Return the stable operation index used for bounded metric-handle lookup.
|
||||
pub fn operation_index(self) -> usize {
|
||||
self.0.metric_index()
|
||||
}
|
||||
|
||||
/// Return the stable S3 operation label associated with this context.
|
||||
pub fn operation_name(self) -> &'static str {
|
||||
self.0.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the current S3 operation, when the task is inside a dispatched S3 request.
|
||||
pub fn current_s3_http_operation_context() -> Option<S3HttpOperationContext> {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
if let Ok(index) = ISOLATED_OPERATION.try_with(Cell::get) {
|
||||
return S3Operation::ALL.get(index).copied().map(S3HttpOperationContext);
|
||||
}
|
||||
CURRENT_OPERATION
|
||||
.try_with(|current| {
|
||||
let index = current.get();
|
||||
S3Operation::ALL.get(index).copied().map(S3HttpOperationContext)
|
||||
})
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// Return the current S3 operation label, or `None` for internal/non-S3 work.
|
||||
pub fn current_s3_http_operation_name() -> Option<&'static str> {
|
||||
current_s3_http_operation_context().map(S3HttpOperationContext::operation_name)
|
||||
}
|
||||
|
||||
/// Reapply a captured S3 operation while polling work on a child Tokio task.
|
||||
pub async fn with_s3_http_operation_context<F>(context: Option<S3HttpOperationContext>, future: F) -> F::Output
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
match context {
|
||||
Some(context) => {
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return with_isolated_s3_http_operation_context(Some(context), future).await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
CURRENT_OPERATION.scope(Cell::new(context.operation_index()), future).await
|
||||
}
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore an item's operation, including explicit absence, without changing
|
||||
/// the ambient HTTP request guard or borrowing the request that flushed a batch.
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
pub async fn with_isolated_s3_http_operation_context<F>(context: Option<S3HttpOperationContext>, future: F) -> F::Output
|
||||
where
|
||||
F: Future,
|
||||
{
|
||||
let index = context.map_or(UNKNOWN_OPERATION, S3HttpOperationContext::operation_index);
|
||||
ISOLATED_OPERATION
|
||||
.scope(Cell::new(index), async {
|
||||
match context {
|
||||
Some(context) => CURRENT_OPERATION.scope(Cell::new(context.operation_index()), future).await,
|
||||
None => future.await,
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
tokio::task_local! {
|
||||
static ISOLATED_OPERATION: Cell<usize>;
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static CURRENT_OPERATION: Cell<usize>;
|
||||
}
|
||||
@@ -240,6 +316,25 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn captured_operation_context_can_be_reapplied_to_child_task() {
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let operation = request
|
||||
.in_scope(|| {
|
||||
observe_s3_http_operation(S3Operation::GetObject);
|
||||
current_s3_http_operation_context()
|
||||
})
|
||||
.expect("dispatched S3 operation should be captured");
|
||||
|
||||
let child_operation = tokio::spawn(with_s3_http_operation_context(Some(operation), async {
|
||||
current_s3_http_operation_name()
|
||||
}))
|
||||
.await
|
||||
.expect("child task should finish");
|
||||
|
||||
assert_eq!(child_operation, Some("s3:GetObject"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outcome_counters_distinguish_partial_and_complete_write_failure() {
|
||||
let counters = HttpOutcomeCounters::new();
|
||||
@@ -279,6 +374,39 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
#[tokio::test]
|
||||
async fn isolated_absent_operation_does_not_inherit_the_flush_request() {
|
||||
let mut request = S3HttpRequestGuard::new("GET");
|
||||
let operation = request.in_scope(|| {
|
||||
observe_s3_http_operation(S3Operation::GetObject);
|
||||
current_s3_http_operation_context().expect("GetObject operation should be captured")
|
||||
});
|
||||
with_s3_http_operation_context(Some(operation), async {
|
||||
assert!(S3HttpRequestGuard::is_active());
|
||||
with_isolated_s3_http_operation_context(None, async {
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
assert!(
|
||||
S3HttpRequestGuard::is_active(),
|
||||
"isolating an item must leave ambient HTTP accounting intact"
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
})
|
||||
.await;
|
||||
assert_eq!(current_s3_http_operation_context(), Some(operation));
|
||||
})
|
||||
.await;
|
||||
assert_eq!(current_s3_http_operation_context(), None);
|
||||
with_isolated_s3_http_operation_context(None, async {
|
||||
assert!(
|
||||
!S3HttpRequestGuard::is_active(),
|
||||
"absent operation must not fabricate an HTTP request guard"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_guard_preserves_operation_across_polls_and_finishes_once() {
|
||||
let totals = || {
|
||||
|
||||
@@ -34,6 +34,20 @@ pub struct Infos {
|
||||
/// enum's crate).
|
||||
const DRIVE_STATE_OK: &str = "ok";
|
||||
|
||||
/// Unresolved delete recovery work, independent of ordinary object healing.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct DeleteRecoveryStatus {
|
||||
pub pending: usize,
|
||||
pub blocked: usize,
|
||||
pub failed: usize,
|
||||
}
|
||||
|
||||
impl DeleteRecoveryStatus {
|
||||
pub fn has_unresolved(&self) -> bool {
|
||||
self.pending != 0 || self.blocked != 0 || self.failed != 0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct HealResultItem {
|
||||
/// Only the coordinator's authenticated full scan may set this. Transported
|
||||
@@ -70,6 +84,8 @@ pub struct HealResultItem {
|
||||
pub resolved_version_id: Option<[u8; 16]>,
|
||||
#[serde(rename = "detail")]
|
||||
pub detail: String,
|
||||
#[serde(default, rename = "deleteRecovery", skip_serializing_if = "Option::is_none")]
|
||||
pub delete_recovery: Option<DeleteRecoveryStatus>,
|
||||
#[serde(rename = "parityBlocks")]
|
||||
pub parity_blocks: usize,
|
||||
#[serde(rename = "dataBlocks")]
|
||||
@@ -118,6 +134,38 @@ impl HealResultItem {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn delete_recovery_status_preserves_partial_results_and_legacy_shape() {
|
||||
let legacy = serde_json::to_value(HealResultItem::default()).expect("serialize legacy result");
|
||||
assert!(legacy.get("deleteRecovery").is_none());
|
||||
let decoded: HealResultItem = serde_json::from_value(legacy).expect("decode result without recovery status");
|
||||
assert!(decoded.delete_recovery.is_none());
|
||||
for status in [
|
||||
DeleteRecoveryStatus {
|
||||
pending: 1,
|
||||
..Default::default()
|
||||
},
|
||||
DeleteRecoveryStatus {
|
||||
blocked: 1,
|
||||
..Default::default()
|
||||
},
|
||||
DeleteRecoveryStatus {
|
||||
failed: 1,
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
let item = HealResultItem {
|
||||
delete_recovery: Some(status),
|
||||
..Default::default()
|
||||
};
|
||||
let wire = serde_json::to_value(item).expect("serialize recovery status");
|
||||
let decoded: HealResultItem = serde_json::from_value(wire).expect("decode recovery status");
|
||||
assert_eq!(decoded.delete_recovery, Some(status));
|
||||
assert!(status.has_unresolved());
|
||||
}
|
||||
assert!(!DeleteRecoveryStatus::default().has_unresolved());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolved_heal_version_is_not_wire_proof() {
|
||||
let item = HealResultItem {
|
||||
|
||||
@@ -38,11 +38,7 @@ impl Serialize for ResourceSet {
|
||||
|
||||
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
|
||||
for resource in &self.0 {
|
||||
let resource_str = match resource {
|
||||
Resource::S3(value) => format!("{}{}", Resource::S3_PREFIX, value),
|
||||
Resource::Kms(value) => format!("{}{}", Resource::KMS_PREFIX, value),
|
||||
};
|
||||
seq.serialize_element(&resource_str)?;
|
||||
seq.serialize_element(resource)?;
|
||||
}
|
||||
seq.end()
|
||||
}
|
||||
@@ -165,6 +161,7 @@ impl<'de> Deserialize<'de> for ResourceSet {
|
||||
|
||||
#[derive(Hash, Eq, PartialEq, Clone, Debug)]
|
||||
pub enum Resource {
|
||||
All,
|
||||
S3(String),
|
||||
Kms(String),
|
||||
}
|
||||
@@ -197,6 +194,7 @@ impl Resource {
|
||||
resolver: Option<&dyn PolicyVariableResolver>,
|
||||
) -> bool {
|
||||
let pattern = match self {
|
||||
Resource::All => return true,
|
||||
Resource::S3(s) => s.to_owned(),
|
||||
Resource::Kms(s) => s.to_owned(),
|
||||
};
|
||||
@@ -242,7 +240,9 @@ impl Resource {
|
||||
impl TryFrom<&str> for Resource {
|
||||
type Error = Error;
|
||||
fn try_from(value: &str) -> std::result::Result<Self, Self::Error> {
|
||||
let resource = if let Some(suffix) = value.strip_prefix(Self::S3_PREFIX) {
|
||||
let resource = if value == "*" {
|
||||
Resource::All
|
||||
} else if let Some(suffix) = value.strip_prefix(Self::S3_PREFIX) {
|
||||
Resource::S3(suffix.into())
|
||||
} else if let Some(suffix) = value.strip_prefix(Self::KMS_PREFIX) {
|
||||
Resource::Kms(suffix.into())
|
||||
@@ -259,6 +259,7 @@ impl Validator for Resource {
|
||||
type Error = Error;
|
||||
fn is_valid(&self) -> std::result::Result<(), Error> {
|
||||
match self {
|
||||
Self::All => {}
|
||||
Self::S3(pattern) => {
|
||||
if pattern.is_empty() || pattern.starts_with('/') {
|
||||
return Err(IamError::InvalidResource("s3".into(), pattern.into()).into());
|
||||
@@ -290,6 +291,7 @@ impl Serialize for Resource {
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
Resource::All => serializer.serialize_str("*"),
|
||||
Resource::S3(s) => serializer.serialize_str(&format!("{}{}", Self::S3_PREFIX, s)),
|
||||
Resource::Kms(s) => serializer.serialize_str(&format!("{}{}", Self::KMS_PREFIX, s)),
|
||||
}
|
||||
@@ -312,6 +314,10 @@ mod tests {
|
||||
use std::collections::HashMap;
|
||||
use test_case::test_case;
|
||||
|
||||
#[test_case("*", "mybucket" => true; "all buckets")]
|
||||
#[test_case("*", "mybucket/myobject" => true; "all objects")]
|
||||
#[test_case("*", "key/mykey" => true; "all keys")]
|
||||
#[test_case("*", "" => true; "unscoped action")]
|
||||
#[test_case("arn:aws:s3:::*","mybucket" => true; "1")]
|
||||
#[test_case("arn:aws:s3:::*","mybucket/myobject" => true; "2")]
|
||||
#[test_case("arn:aws:s3:::mybucket*","mybucket" => true; "3")]
|
||||
@@ -346,6 +352,32 @@ mod tests {
|
||||
pollster::block_on(resource.is_match(object, &HashMap::new()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wildcard_resource_serialization_round_trip() {
|
||||
use crate::policy::resource::ResourceSet;
|
||||
|
||||
let resource: Resource = serde_json::from_str(r#""*""#).expect("wildcard resource should parse");
|
||||
assert_eq!(serde_json::to_string(&resource).expect("resource should serialize"), r#""*""#);
|
||||
|
||||
for json in [r#""*""#, r#"["*"]"#, r#"["*","*"]"#] {
|
||||
let set: ResourceSet = serde_json::from_str(json).expect("wildcard resource set should parse");
|
||||
assert_eq!(set.len(), 1, "duplicate wildcard resources should be removed");
|
||||
let serialized = serde_json::to_string(&set).expect("resource set should serialize");
|
||||
assert_eq!(serialized, r#"["*"]"#);
|
||||
let round_trip: ResourceSet = serde_json::from_str(&serialized).expect("resource set should parse again");
|
||||
assert_eq!(round_trip, set);
|
||||
}
|
||||
}
|
||||
|
||||
#[test_case(""; "empty resource")]
|
||||
#[test_case("**"; "multiple wildcards")]
|
||||
#[test_case("*/*"; "unqualified path")]
|
||||
#[test_case(" *"; "leading whitespace")]
|
||||
#[test_case("arn:aws:unknown:::*"; "unknown service")]
|
||||
fn test_invalid_resource_is_rejected(resource: &str) {
|
||||
assert!(Resource::try_from(resource).is_err(), "invalid resource must not become a wildcard");
|
||||
}
|
||||
|
||||
#[test_case("arn:aws:kms:::key/mykey" => true; "key id parses")]
|
||||
#[test_case("arn:aws:kms:::key/*" => true; "key wildcard parses")]
|
||||
#[test_case("arn:aws:kms:::key/app-key.v2" => true; "key id with dot parses")]
|
||||
|
||||
@@ -182,15 +182,24 @@ impl Statement {
|
||||
/// left empty. An empty `args.object` means the caller did not scope the request to a
|
||||
/// key, which preserves the legacy match-every-key behaviour.
|
||||
async fn kms_key_scope_matches(&self, args: &Args<'_>, resolver: &VariableResolver) -> bool {
|
||||
if self.not_resources.iter().any(|resource| matches!(resource, Resource::All)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if matches!(args.action, Action::KmsAction(KmsAction::BackupAction | KmsAction::RestoreAction))
|
||||
&& (!self.resources.is_empty() || !self.not_resources.is_empty())
|
||||
&& ((!self.resources.is_empty() && !self.resources.iter().any(|resource| matches!(resource, Resource::All)))
|
||||
|| !self.not_resources.is_empty())
|
||||
{
|
||||
// Global bundle operations require an unscoped Allow, while a Deny
|
||||
// covering any key must still block an operation covering every key.
|
||||
return matches!(self.effect, Effect::Deny);
|
||||
}
|
||||
|
||||
let kms_resources: Vec<&Resource> = self.resources.iter().filter(|resource| resource.is_kms()).collect();
|
||||
let kms_resources: Vec<&Resource> = self
|
||||
.resources
|
||||
.iter()
|
||||
.filter(|resource| matches!(resource, Resource::All | Resource::Kms(_)))
|
||||
.collect();
|
||||
let kms_not_resources: Vec<&Resource> = self.not_resources.iter().filter(|resource| resource.is_kms()).collect();
|
||||
|
||||
if kms_resources.len() != self.resources.len() || kms_not_resources.len() != self.not_resources.len() {
|
||||
|
||||
@@ -808,3 +808,128 @@ fn merge_keeps_allow_statements_differing_only_in_not_resource() {
|
||||
"both Allow statements must survive the merge; neither may be dropped as a duplicate"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wildcard_deny_only_session_grants_no_storage_access() {
|
||||
let policy = Policy::parse_config(
|
||||
br#"{"Statement":[{"Action":["s3:GetObject"],"Effect":"Deny","Resource":"*","Sid":"1"}],"Version":"2012-10-17"}"#,
|
||||
)
|
||||
.expect("deny-only session policy should parse");
|
||||
let encoded = serde_json::to_vec(&policy).expect("session policy should serialize into claims");
|
||||
let decoded = Policy::parse_config(&encoded).expect("session policy should parse from claims");
|
||||
for action in ["s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucket"] {
|
||||
for bucket in ["bucket", "other-bucket"] {
|
||||
assert!(
|
||||
!policy_is_allowed(
|
||||
decoded.clone(),
|
||||
ArgsBuilder {
|
||||
action: action.into(),
|
||||
bucket: bucket.into(),
|
||||
object: "data/object".into(),
|
||||
..Default::default()
|
||||
}
|
||||
),
|
||||
"deny-only session must not grant {action} on {bucket}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test_case("s3:GetObject", "bucket", "object" => true; "matching object action")]
|
||||
#[test_case("s3:PutObject", "bucket", "object" => false; "ungranted action")]
|
||||
#[test_case("s3:GetObject", "other-bucket", "object" => true; "other bucket")]
|
||||
fn wildcard_resource_allow_preserves_action_scope(action: &str, bucket: &str, object: &str) -> bool {
|
||||
let policy = Policy::parse_config(
|
||||
br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":["*"]}]}"#,
|
||||
)
|
||||
.expect("wildcard allow policy should parse");
|
||||
policy_is_allowed(
|
||||
policy,
|
||||
ArgsBuilder {
|
||||
action: action.into(),
|
||||
bucket: bucket.into(),
|
||||
object: object.into(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[test_case("s3:GetObject", "bucket", "object"; "object")]
|
||||
#[test_case("s3:ListBucket", "bucket", ""; "bucket")]
|
||||
#[test_case("s3:ListAllMyBuckets", "", ""; "all buckets")]
|
||||
#[test_case("kms:Decrypt", "", "mykey"; "kms key")]
|
||||
#[test_case("kms:Backup", "", ""; "kms backup")]
|
||||
#[test_case("kms:Restore", "", ""; "kms restore")]
|
||||
fn wildcard_resource_allow_deny_and_exclusion(action: &str, bucket: &str, object: &str) {
|
||||
let other_resource = if action.starts_with("kms:") {
|
||||
"arn:aws:kms:::key/other-key"
|
||||
} else {
|
||||
"arn:aws:s3:::other-bucket/*"
|
||||
};
|
||||
for (field, resource) in [
|
||||
("Resource", serde_json::json!("*")),
|
||||
("Resource", serde_json::json!(["*"])),
|
||||
("Resource", serde_json::json!([other_resource, "*"])),
|
||||
("NotResource", serde_json::json!("*")),
|
||||
("NotResource", serde_json::json!([other_resource, "*"])),
|
||||
] {
|
||||
let mut json = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{"Effect": "Allow", "Action": action, field: resource}]
|
||||
});
|
||||
let allow = Policy::parse_config(&serde_json::to_vec(&json).expect("test policy should serialize"))
|
||||
.expect("wildcard policy should parse");
|
||||
let args = ArgsBuilder {
|
||||
action: action.into(),
|
||||
bucket: bucket.into(),
|
||||
object: object.into(),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(policy_is_allowed(allow, args), field == "Resource", "{action} with {field}");
|
||||
|
||||
json["Statement"][0]["Effect"] = serde_json::json!("Deny");
|
||||
json["Statement"]
|
||||
.as_array_mut()
|
||||
.expect("statements are an array")
|
||||
.push(serde_json::json!({
|
||||
"Effect": "Allow", "Action": action, "Resource": "*"
|
||||
}));
|
||||
let deny = Policy::parse_config(&serde_json::to_vec(&json).expect("test policy should serialize"))
|
||||
.expect("wildcard deny policy should parse");
|
||||
assert_eq!(
|
||||
policy_is_allowed(
|
||||
deny,
|
||||
ArgsBuilder {
|
||||
action: action.into(),
|
||||
bucket: bucket.into(),
|
||||
object: object.into(),
|
||||
..Default::default()
|
||||
}
|
||||
),
|
||||
field == "NotResource",
|
||||
"explicit deny must win unless {field} excludes the resource for {action}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test_case(Some("allowed/data/") => true; "matching prefix")]
|
||||
#[test_case(Some("other/data/") => false; "outside allowed prefix")]
|
||||
#[test_case(None => false; "missing prefix")]
|
||||
fn wildcard_resource_preserves_conditions(prefix: Option<&str>) -> bool {
|
||||
let policy = Policy::parse_config(
|
||||
br#"{"Version":"2012-10-17","Statement":[{
|
||||
"Effect":"Allow","Action":"s3:ListBucket","Resource":"*",
|
||||
"Condition":{"StringLike":{"s3:prefix":"allowed/*"}}
|
||||
}]}"#,
|
||||
)
|
||||
.expect("conditional wildcard policy should parse");
|
||||
let mut args = ArgsBuilder {
|
||||
action: "s3:ListBucket".into(),
|
||||
bucket: "bucket".into(),
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(prefix) = prefix {
|
||||
args.conditions.insert("prefix".into(), vec![prefix.into()]);
|
||||
}
|
||||
policy_is_allowed(policy, args)
|
||||
}
|
||||
|
||||
@@ -785,6 +785,18 @@ pub struct DeleteVersionsRequest {
|
||||
#[prost(bytes = "bytes", tag = "6")]
|
||||
pub opts_bin: ::prost::bytes::Bytes,
|
||||
}
|
||||
/// Additive, fail-closed conditional batch protocol. Probe requests never mutate disk state.
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
pub struct GuardedDeleteVersionsRequest {
|
||||
#[prost(string, tag = "1")]
|
||||
pub disk: ::prost::alloc::string::String,
|
||||
#[prost(string, tag = "2")]
|
||||
pub volume: ::prost::alloc::string::String,
|
||||
#[prost(bytes = "bytes", tag = "3")]
|
||||
pub payload: ::prost::bytes::Bytes,
|
||||
#[prost(bool, tag = "4")]
|
||||
pub probe_only: bool,
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct DeleteVersionsResponse {
|
||||
#[prost(bool, tag = "1")]
|
||||
@@ -2135,6 +2147,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "AcquireSnapshotLease"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn acquire_quota_mutation_fence(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SnapshotLeaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/AcquireQuotaMutationFence");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "AcquireQuotaMutationFence"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn renew_snapshot_lease(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SnapshotLeaseRenewRequest>,
|
||||
@@ -2165,6 +2192,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ReleaseSnapshotLease"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn release_quota_mutation_fence(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::SnapshotLeaseReleaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseMutationResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReleaseQuotaMutationFence");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "ReleaseQuotaMutationFence"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn verify_file(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::VerifyFileRequest>,
|
||||
@@ -2376,6 +2418,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenameDataAtIncarnation"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn rename_data_with_quota_fence(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::RenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameDataWithQuotaFence");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "RenameDataWithQuotaFence"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn make_volumes(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::MakeVolumesRequest>,
|
||||
@@ -2616,6 +2673,21 @@ pub mod node_service_client {
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "DeleteVersions"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn guarded_delete_versions(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::GuardedDeleteVersionsRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::DeleteVersionsResponse>, tonic::Status> {
|
||||
self.inner
|
||||
.ready()
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unknown(format!("Service was not ready: {}", e.into())))?;
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/GuardedDeleteVersions");
|
||||
let mut req = request.into_request();
|
||||
req.extensions_mut()
|
||||
.insert(GrpcMethod::new("node_service.NodeService", "GuardedDeleteVersions"));
|
||||
self.inner.unary(req, path, codec).await
|
||||
}
|
||||
pub async fn read_multiple(
|
||||
&mut self,
|
||||
request: impl tonic::IntoRequest<super::ReadMultipleRequest>,
|
||||
@@ -3509,6 +3581,10 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::SnapshotLeaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status>;
|
||||
async fn acquire_quota_mutation_fence(
|
||||
&self,
|
||||
request: tonic::Request<super::SnapshotLeaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseResponse>, tonic::Status>;
|
||||
async fn renew_snapshot_lease(
|
||||
&self,
|
||||
request: tonic::Request<super::SnapshotLeaseRenewRequest>,
|
||||
@@ -3517,6 +3593,10 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::SnapshotLeaseReleaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseMutationResponse>, tonic::Status>;
|
||||
async fn release_quota_mutation_fence(
|
||||
&self,
|
||||
request: tonic::Request<super::SnapshotLeaseReleaseRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::SnapshotLeaseMutationResponse>, tonic::Status>;
|
||||
async fn verify_file(
|
||||
&self,
|
||||
request: tonic::Request<super::VerifyFileRequest>,
|
||||
@@ -3586,6 +3666,10 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status>;
|
||||
async fn rename_data_with_quota_fence(
|
||||
&self,
|
||||
request: tonic::Request<super::RenameDataRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::RenameDataResponse>, tonic::Status>;
|
||||
async fn make_volumes(
|
||||
&self,
|
||||
request: tonic::Request<super::MakeVolumesRequest>,
|
||||
@@ -3650,6 +3734,10 @@ pub mod node_service_server {
|
||||
&self,
|
||||
request: tonic::Request<super::DeleteVersionsRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::DeleteVersionsResponse>, tonic::Status>;
|
||||
async fn guarded_delete_versions(
|
||||
&self,
|
||||
request: tonic::Request<super::GuardedDeleteVersionsRequest>,
|
||||
) -> std::result::Result<tonic::Response<super::DeleteVersionsResponse>, tonic::Status>;
|
||||
async fn read_multiple(
|
||||
&self,
|
||||
request: tonic::Request<super::ReadMultipleRequest>,
|
||||
@@ -4335,6 +4423,34 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/AcquireQuotaMutationFence" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct AcquireQuotaMutationFenceSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::SnapshotLeaseRequest> for AcquireQuotaMutationFenceSvc<T> {
|
||||
type Response = super::SnapshotLeaseResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::SnapshotLeaseRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::acquire_quota_mutation_fence(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = AcquireQuotaMutationFenceSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenewSnapshotLease" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenewSnapshotLeaseSvc<T: NodeService>(pub Arc<T>);
|
||||
@@ -4391,6 +4507,34 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/ReleaseQuotaMutationFence" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ReleaseQuotaMutationFenceSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::SnapshotLeaseReleaseRequest> for ReleaseQuotaMutationFenceSvc<T> {
|
||||
type Response = super::SnapshotLeaseMutationResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::SnapshotLeaseReleaseRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::release_quota_mutation_fence(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = ReleaseQuotaMutationFenceSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/VerifyFile" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct VerifyFileSvc<T: NodeService>(pub Arc<T>);
|
||||
@@ -4786,6 +4930,34 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/RenameDataWithQuotaFence" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct RenameDataWithQuotaFenceSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::RenameDataRequest> for RenameDataWithQuotaFenceSvc<T> {
|
||||
type Response = super::RenameDataResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::RenameDataRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::rename_data_with_quota_fence(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = RenameDataWithQuotaFenceSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/MakeVolumes" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct MakeVolumesSvc<T: NodeService>(pub Arc<T>);
|
||||
@@ -5234,6 +5406,34 @@ pub mod node_service_server {
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/GuardedDeleteVersions" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct GuardedDeleteVersionsSvc<T: NodeService>(pub Arc<T>);
|
||||
impl<T: NodeService> tonic::server::UnaryService<super::GuardedDeleteVersionsRequest> for GuardedDeleteVersionsSvc<T> {
|
||||
type Response = super::DeleteVersionsResponse;
|
||||
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
|
||||
fn call(&mut self, request: tonic::Request<super::GuardedDeleteVersionsRequest>) -> Self::Future {
|
||||
let inner = Arc::clone(&self.0);
|
||||
let fut = async move { <T as NodeService>::guarded_delete_versions(&inner, request).await };
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
let accept_compression_encodings = self.accept_compression_encodings;
|
||||
let send_compression_encodings = self.send_compression_encodings;
|
||||
let max_decoding_message_size = self.max_decoding_message_size;
|
||||
let max_encoding_message_size = self.max_encoding_message_size;
|
||||
let inner = self.inner.clone();
|
||||
let fut = async move {
|
||||
let method = GuardedDeleteVersionsSvc(inner);
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec)
|
||||
.apply_compression_config(accept_compression_encodings, send_compression_encodings)
|
||||
.apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size);
|
||||
let res = grpc.unary(method, req).await;
|
||||
Ok(res)
|
||||
};
|
||||
Box::pin(fut)
|
||||
}
|
||||
"/node_service.NodeService/ReadMultiple" => {
|
||||
#[allow(non_camel_case_types)]
|
||||
struct ReadMultipleSvc<T: NodeService>(pub Arc<T>);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user