mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-04 01:53:36 +00:00
Compare commits
30 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 116a5299b2 | |||
| fbc54f7212 | |||
| 3aa492f5ee | |||
| 00a81b4048 | |||
| 20ef9ebf15 | |||
| cc08e0d11b | |||
| 69294523d5 | |||
| cd4d86f28a | |||
| 2f55cf69e1 | |||
| fff6e94378 | |||
| 41b13aeb8b | |||
| 88587a8f94 | |||
| 8c8b5da63f | |||
| 00a238f838 | |||
| 965d805a4f | |||
| 9d66e04ae3 | |||
| 12c58d25b5 | |||
| 1ac1d883f4 | |||
| 45ba33291a | |||
| 636eded277 | |||
| ed4099ebe1 | |||
| 3a1fac16e3 | |||
| b00e85f1a4 | |||
| ecbddc188a | |||
| 2d63931bba | |||
| d429c6f792 | |||
| e494dc2b4b | |||
| 2b7c927ee8 | |||
| 647846956f | |||
| 04690f7da5 |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-linux=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
sha256-darwin=2f97cbbeb2dbf52919b704a3d07336c162a41f95079d941d74251f9279c494e6
|
||||
|
||||
+12
-6
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# Nightly and manual-dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
@@ -970,10 +970,16 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# Keep storage-heavy cases isolated in the full suite too. The large pagination
|
||||
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
|
||||
# Keep the storage-heavy smoke cases isolated in the full suite too. They
|
||||
# start multiple server fixtures and issue many sequential S3 operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
|
||||
# Reserve the runner so other server fixtures do not starve its disk operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
|
||||
@@ -32,6 +32,8 @@ updates:
|
||||
- "overtrue"
|
||||
- "majinghe"
|
||||
ignore:
|
||||
- dependency-name: "pulsar"
|
||||
versions: [ "6.9.x" ]
|
||||
- dependency-name: "object_store"
|
||||
versions: [ "0.13.x" ]
|
||||
- dependency-name: "libunftp"
|
||||
|
||||
@@ -296,6 +296,32 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
|
||||
"name": "RustFS Validation Sweep",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"30 19 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 60,
|
||||
"result_owner": "rustfs-validation-sweep.yml",
|
||||
"disposition": "canonical validation cases executed in rotating batches",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/btf-prototype-reference.yml",
|
||||
"name": "BTF prototype reference",
|
||||
@@ -743,12 +769,10 @@
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"name": "e2e-distributed",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"workflow_dispatch",
|
||||
"schedule"
|
||||
],
|
||||
@@ -823,12 +847,11 @@
|
||||
"workflow": ".github/workflows/e2e-upgrade.yml",
|
||||
"name": "Upgrade Compatibility",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"push",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -497,6 +497,63 @@ jobs:
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
fips-provider:
|
||||
name: FIPS provider boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [default, fips]
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-fips-provider
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install FIPS build tools
|
||||
if: matrix.mode == 'fips'
|
||||
run: sudo apt-get install -y cmake golang clang libclang-dev
|
||||
|
||||
- name: Verify provider installation
|
||||
shell: bash
|
||||
run: |
|
||||
args=()
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
args+=(--features fips)
|
||||
fi
|
||||
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
|
||||
fi
|
||||
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
|
||||
fi
|
||||
for test_name in "${tests[@]}"; do
|
||||
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
|
||||
done
|
||||
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
@@ -1397,6 +1454,7 @@ jobs:
|
||||
- workspace-shard-evidence
|
||||
- core-contract
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
@@ -1424,6 +1482,7 @@ jobs:
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
|
||||
# and manual dispatches execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
@@ -33,32 +33,6 @@
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".config/e2e-distributed-selection.txt"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
|
||||
@@ -12,30 +12,19 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Single-node upgrade and rollback lane.
|
||||
#
|
||||
# The weekly Monday schedule, version tags, and workflow_dispatch run this
|
||||
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
|
||||
|
||||
name: Upgrade Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/distributed/**"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/rpc_fault_proxy.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/storage-api/**"
|
||||
- "rustfs/**"
|
||||
- "Cargo.lock"
|
||||
push:
|
||||
tags:
|
||||
- "[0-9]*.[0-9]*.[0-9]*"
|
||||
schedule:
|
||||
# 03:17 UTC every Monday.
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -44,7 +33,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -346,12 +346,12 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
# The dispatch below runs only after this upload and the immutable R2
|
||||
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
|
||||
# which still starts from this workflow's workflow_run completion.
|
||||
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -360,6 +360,90 @@ jobs:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
|
||||
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
|
||||
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
|
||||
# or fine-grained Contents write (the permission GitHub checks for
|
||||
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
|
||||
# step, and therefore this job, so the scheduled failure alert fires.
|
||||
# GitHub allows at most 10 top-level client_payload properties, so the
|
||||
# schema 2 candidate object is nested under `candidate` and the release
|
||||
# fields sit beside it.
|
||||
- name: Dispatch nightly candidate to auto-testing
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
|
||||
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
|
||||
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
|
||||
: "${DEB_DATE:?DEB_DATE is required}"
|
||||
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
|
||||
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
|
||||
|
||||
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
|
||||
--arg event_type "nightly-candidate-ready" \
|
||||
--arg source_repo "${SOURCE_REPO}" \
|
||||
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
|
||||
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
|
||||
--arg channel "nightly" \
|
||||
--arg deb_date "${DEB_DATE}" \
|
||||
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
source_repo: $source_repo,
|
||||
latest_alias_url: $latest_alias_url,
|
||||
auto_testing_release: $auto_testing_release,
|
||||
channel: $channel,
|
||||
deb_date: $deb_date,
|
||||
nightly_workflow: $nightly_workflow,
|
||||
candidate: $candidate[0]
|
||||
}' > "${payload}"
|
||||
if ! jq -e '
|
||||
.event_type == "nightly-candidate-ready"
|
||||
and .source_repo == "rustfs/rustfs"
|
||||
and .channel == "nightly"
|
||||
and .deb_date == env.DEB_DATE
|
||||
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
|
||||
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
|
||||
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
|
||||
and .candidate.schema == 2
|
||||
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
|
||||
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
|
||||
' "${payload}" >/dev/null; then
|
||||
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile client_payload "${payload}" \
|
||||
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
|
||||
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
|
||||
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
|
||||
-o "${response}" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"${body}" \
|
||||
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
|
||||
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ "${status}" != "204" ]]; then
|
||||
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
|
||||
if [[ -s "${response}" ]]; then
|
||||
cat "${response}" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Validation Sweep
|
||||
|
||||
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
|
||||
# on the shared functional fleet. Everything with a lane executor is swept in
|
||||
# rotating nightly batches so the whole set is covered roughly weekly; a
|
||||
# single case or one module can be dispatched manually. That is the
|
||||
# implemented catalog (66 today) plus the eight planned cases whose dedicated
|
||||
# executors already exist and have live PASS evidence - fresh nightly
|
||||
# evidence feeds their acceptance review. heal.* cases stay on the
|
||||
# interactive validation flow: they require reviewed native/tool collectors
|
||||
# built by separate validation-tools runs before fleet execution.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# After the nightly functional chain settles (chain starts ~16:48 UTC and
|
||||
# runs for about two hours).
|
||||
- cron: "30 19 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
module:
|
||||
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
limit:
|
||||
description: 'Maximum cases per sweep (0 = run everything selected)'
|
||||
type: number
|
||||
required: false
|
||||
default: 12
|
||||
build_run_id:
|
||||
description: 'Successful nightly package run; empty resolves the latest'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: 'Attempt of that package run'
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-sweep
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve nightly candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scheduled runs carry no inputs and the resolver only derives a
|
||||
# build from a workflow_run payload (which a schedule cannot
|
||||
# provide), so resolve the newest successful nightly build
|
||||
# explicitly before invoking it.
|
||||
if [ -z "${BUILD_RUN_ID:-}" ]; then
|
||||
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
|
||||
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
|
||||
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
|
||||
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
|
||||
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
|
||||
fi
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = chain["candidate"]["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + chain["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.candidate.outputs.testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
# The pool / ilm / replication validation runners require a managed
|
||||
# Python, exactly like the chain's validation job provides.
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python requirements
|
||||
# Case scripts call python3 and import boto3 at startup. Install the
|
||||
# functional chain's client into a per-run env so that interpreter
|
||||
# sees it without changing the runner's shared Python.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
SELECTED_CASE: ${{ inputs.case }}
|
||||
SELECTED_MODULE: ${{ inputs.module }}
|
||||
CASE_LIMIT: ${{ inputs.limit }}
|
||||
SWEEP_EVENT: ${{ github.event_name }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
sweep_root = Path(os.environ["SWEEP_ROOT"])
|
||||
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
def sweepable(case):
|
||||
status = case.get("source_status")
|
||||
if status == "implemented":
|
||||
return True
|
||||
# The remaining planned non-heal cases route to the lane
|
||||
# executors and already have live PASS evidence; sweeping them
|
||||
# nightly produces fresh evidence for their acceptance review.
|
||||
return status == "planned" and not case["id"].startswith("heal.")
|
||||
|
||||
selected = [(case["id"], module["id"]) for module in catalog["modules"]
|
||||
for case in module["cases"]
|
||||
if sweepable(case)]
|
||||
if not selected:
|
||||
raise SystemExit("catalog has no sweepable cases")
|
||||
|
||||
one_case = os.environ.get("SELECTED_CASE", "").strip()
|
||||
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
|
||||
known = {case_id for case_id, _ in selected}
|
||||
modules = {module_id for _, module_id in selected}
|
||||
if one_case:
|
||||
if one_case not in known:
|
||||
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
|
||||
selected = [(one_case, dict(selected)[one_case])]
|
||||
elif module_filter:
|
||||
if module_filter not in modules:
|
||||
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
|
||||
selected = [pair for pair in selected if pair[1] == module_filter]
|
||||
elif os.environ.get("SWEEP_EVENT") == "schedule":
|
||||
# Stateless rotation: every night take one window of the catalog
|
||||
# so a full sweep needs len(selected)/limit nights and no state
|
||||
# has to be carried between runs.
|
||||
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
|
||||
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
|
||||
start = (day * limit) % len(selected)
|
||||
selected = (selected[start:] + selected[:start])[:limit]
|
||||
limit = int(os.environ.get("CASE_LIMIT") or 0)
|
||||
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
|
||||
if limit > 0:
|
||||
selected = selected[:limit]
|
||||
|
||||
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"count={len(selected)}\n")
|
||||
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
|
||||
PY
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Run validation cases
|
||||
id: run_cases
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p "${SWEEP_ROOT}/results"
|
||||
: > "${SWEEP_ROOT}/results.txt"
|
||||
while read -r CASE_ID; do
|
||||
[ -n "${CASE_ID}" ] || continue
|
||||
echo "::group::case ${CASE_ID}"
|
||||
rm -rf "${SWEEP_ROOT}/run"
|
||||
if ./auto-testing/scripts/run_validation_case.sh \
|
||||
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
|
||||
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
|
||||
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: PASS"
|
||||
else
|
||||
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
|
||||
fi
|
||||
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
|
||||
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
# Fleet hygiene between cases: the case runners set up their own
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
# -n keeps ssh from consuming the case list this loop reads from stdin
|
||||
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "inter-case cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
done < "${SWEEP_ROOT}/sweep-cases.txt"
|
||||
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
|
||||
|
||||
- name: Generate sweep report
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["SWEEP_ROOT"])
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
|
||||
rows = []
|
||||
for line in (root / "results.txt").read_text().splitlines():
|
||||
if not line:
|
||||
continue
|
||||
case_id, status = line.split("\t")
|
||||
rows.append((case_id, module_of.get(case_id, "?"), status))
|
||||
passed = sum(1 for _, _, s in rows if s == "PASS")
|
||||
failed = len(rows) - passed
|
||||
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
|
||||
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
|
||||
f"{os.environ.get('GITHUB_RUN_ID', '')}")
|
||||
|
||||
report = [
|
||||
"# RustFS validation sweep report", "",
|
||||
f"- Run: {run_url}",
|
||||
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
|
||||
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
|
||||
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
|
||||
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
|
||||
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
|
||||
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
|
||||
]
|
||||
# Group the executed cases by catalog module (the major category:
|
||||
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
|
||||
# data services, identity, s3 compatibility). Per-module subtotals
|
||||
# deliberately use "- Subtotal:" so the dashboard overview regexes
|
||||
# (which anchor on "- Total:") keep reading the grand totals below.
|
||||
by_module = {}
|
||||
module_order = []
|
||||
for case_id, module, status in rows:
|
||||
if module not in by_module:
|
||||
by_module[module] = []
|
||||
module_order.append(module)
|
||||
by_module[module].append((case_id, status))
|
||||
report += ["## Case Summary", ""]
|
||||
for module in module_order:
|
||||
cases = by_module[module]
|
||||
mod_pass = sum(1 for _, st in cases if st == "PASS")
|
||||
mod_fail = len(cases) - mod_pass
|
||||
report += [f"### {module} ({len(cases)})", "",
|
||||
"| Case | Status |", "| --- | --- |"]
|
||||
report += [f"| {case_id} | {status} |" for case_id, status in cases]
|
||||
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
|
||||
report += [
|
||||
"## Totals", "",
|
||||
f"- Total: {len(rows)}",
|
||||
f"- PASS: {passed}",
|
||||
f"- FAIL: {failed}",
|
||||
"- UNSUPPORTED: 0", "",
|
||||
"Failing cases keep this workflow red only as a report; the sweep artifact",
|
||||
"retains every case log and per-case report for diagnosis.", "",
|
||||
]
|
||||
(root / "report.md").write_text("\n".join(report))
|
||||
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
|
||||
PY
|
||||
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
|
||||
SUITE: validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Retain sweep evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/validation-sweep/report.md
|
||||
${{ runner.temp }}/validation-sweep/results.txt
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
${{ runner.temp }}/validation-sweep/run/
|
||||
# warn (not error): an early infra failure legitimately leaves no
|
||||
# per-case evidence, and the failing step is already red.
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
|
||||
|
||||
- name: Enforce sweep result
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
|
||||
echo "validation sweep has failing cases:"
|
||||
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
# A scheduled sweep failure usually means a canonical validation case
|
||||
# regressed or the fleet environment broke; surface it as a tracking
|
||||
# issue instead of leaving it in the Actions tab.
|
||||
needs: [sweep]
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
|
||||
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
|
||||
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
|
||||
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
|
||||
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
|
||||
|
||||
### Security
|
||||
|
||||
Generated
+379
-179
File diff suppressed because it is too large
Load Diff
+12
-15
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -212,12 +212,13 @@ serde_urlencoded = "0.7.1"
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
aws-lc-rs = { version = "1.18.0", default-features = false }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
@@ -234,7 +235,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.38" }
|
||||
jiff = { version = "0.2.39" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -253,9 +254,9 @@ atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.123.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.129.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.124.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.19.2" }
|
||||
@@ -337,7 +338,7 @@ strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.0"
|
||||
test-case = "3.4.1"
|
||||
thiserror = "2.0.21"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
@@ -356,13 +357,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.1.3"
|
||||
zip = "9.3.0"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.6", default-features = false }
|
||||
dial9 = { version = "0.5.7", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -384,9 +385,9 @@ russh-sftp = "3.0.1"
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.2", default-features = false }
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -427,7 +428,3 @@ strip = "none"
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
|
||||
|
||||
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
|
||||
@@ -12,11 +12,249 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use std::any::TypeId;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: Debug + ProvideErrorMetadata + 'static,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
// The workload uses unique unversioned keys. Repeat only explicitly
|
||||
// advertised lock backpressure; every step still needs its real result.
|
||||
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().is_some_and(|error| {
|
||||
let Some(response) = error.raw_response() else {
|
||||
return false;
|
||||
};
|
||||
let Some(service_error) = error.as_service_error() else {
|
||||
return false;
|
||||
};
|
||||
// HEAD has no XML error body, so its SDK service error can
|
||||
// lack a code even when the server supplies retry headers.
|
||||
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|
||||
|| (service_error.code().is_none()
|
||||
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
|
||||
response.status().as_u16() == 503
|
||||
&& namespace_code
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
&& response.headers().get("retry-after") == Some("1")
|
||||
});
|
||||
if !retry {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
|
||||
eprintln!(
|
||||
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
outcome.map_err(|_| {
|
||||
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
|
||||
let actual = output.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != expected {
|
||||
return Err(format!(
|
||||
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
|
||||
expected.len(),
|
||||
sha256_hex(expected),
|
||||
actual.len(),
|
||||
sha256_hex(&actual)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_http_client::test_util::infallible_client_fn;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
|
||||
|
||||
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let transport = infallible_client_fn(move |_| {
|
||||
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
|
||||
let body = if code.is_empty() {
|
||||
body.to_string()
|
||||
} else {
|
||||
format!("<Error><Code>{code}</Code></Error>")
|
||||
};
|
||||
let mut response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len());
|
||||
if let Some(cause) = cause {
|
||||
response = response.header("x-rustfs-error-cause", cause);
|
||||
}
|
||||
if let Some(retry_after) = retry_after {
|
||||
response = response.header("retry-after", retry_after);
|
||||
}
|
||||
response.body(SdkBody::from(body)).expect("scripted S3 response")
|
||||
});
|
||||
let config = crate::common::build_test_s3_config(
|
||||
"http://contention.example.invalid",
|
||||
"synthetic-access",
|
||||
"synthetic-secret",
|
||||
None,
|
||||
"contention-test",
|
||||
);
|
||||
let client = Client::from_conf(config.to_builder().http_client(transport).build());
|
||||
(s3_probe_client(&client), requests)
|
||||
}
|
||||
|
||||
fn contention() -> Reply {
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt < 3 {
|
||||
contention()
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
let standard = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
|
||||
.build(),
|
||||
);
|
||||
standard
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("three advertised lock responses exhaust the standard SDK policy");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 3);
|
||||
requests.store(0, Ordering::SeqCst);
|
||||
let started = Instant::now();
|
||||
after_lock_contention("PUT recovery", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt == 0 {
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
|
||||
for reply in [
|
||||
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(403, "AccessDenied", None, None, ""),
|
||||
(404, "NoSuchKey", None, None, ""),
|
||||
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
|
||||
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "ServiceUnavailable", None, Some("1"), ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
|
||||
] {
|
||||
let (client, requests) = client(move |_| reply);
|
||||
let result = after_lock_contention("PUT fatal response", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await?;
|
||||
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistent_lock_contention_exhausts_the_request_budget() {
|
||||
let (client, requests) = client(|_| contention());
|
||||
let started = Instant::now();
|
||||
let result = after_lock_contention("PUT unavailable", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
|
||||
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
|
||||
assert!(requests.load(Ordering::SeqCst) > 1);
|
||||
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_wrong_bytes_fail_without_retry() {
|
||||
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
|
||||
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
|
||||
.await
|
||||
.expect_err("a successful but incorrect read is a consistency failure");
|
||||
assert!(error.to_string().contains("bytes mismatch"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
// This consistency probe owns its admission policy; unrelated responses
|
||||
// must not be hidden by the SDK's general-purpose retries.
|
||||
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
let context = format!("worker={idx} round={round}");
|
||||
after_lock_contention(&format!("{context} PUT source"), || {
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
let head = after_lock_contention(&format!("{context} HEAD source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
assert_workload_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
after_lock_contention(&format!("{context} COPY retained"), || {
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_workload_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
after_lock_contention(&format!("{context} DELETE source"), || {
|
||||
writer.delete_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
assert_eq!(
|
||||
inventory.len(),
|
||||
worker_count * rounds,
|
||||
"every worker round must retain its acknowledged copy"
|
||||
);
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
|
||||
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
|
||||
})
|
||||
.await??;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
assert_workload_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
|
||||
@@ -576,34 +576,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn activate_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn cancel_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_reserved_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data_with_pool_meta_cas(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
|
||||
ServerSideEncryption, VersioningConfiguration,
|
||||
};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let source_log = format!("{}/source.log", source_env.temp_dir);
|
||||
source_env.capture_log_path = Some(source_log.clone());
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
"target did not preserve the source delete-marker version ID"
|
||||
);
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
let wait_for_source_writeback = || async {
|
||||
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let response = signed_request(
|
||||
http::Method::GET,
|
||||
&metrics_url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
|
||||
let queued = metrics["queued"]["curr"]["count"]
|
||||
.as_f64()
|
||||
.ok_or("missing replication queue count")?;
|
||||
if metrics["provider_available"] == true && queued == 0.0 {
|
||||
return Ok::<(), Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
};
|
||||
wait_for_source_writeback().await?;
|
||||
|
||||
let purged = source_client
|
||||
.delete_objects()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
|
||||
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != purged_version_id),
|
||||
"target retained the explicitly purged object version"
|
||||
);
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete marker"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
// Wait for the source workers to finish their final writeback, which runs
|
||||
// after the remote purge has already become visible (rustfs#9121).
|
||||
wait_for_source_writeback().await?;
|
||||
let logs = fs::read_to_string(source_log).await?;
|
||||
let writeback_failures: Vec<_> = logs
|
||||
.lines()
|
||||
.filter(|line| line.contains("apply_replication_delete_state"))
|
||||
.collect();
|
||||
assert!(
|
||||
writeback_failures.is_empty(),
|
||||
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
|
||||
@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -690,7 +691,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -5209,6 +5210,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
|
||||
@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
|
||||
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
@@ -116,6 +117,10 @@ mod tests {
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
"config/replication/mrf.bin.recovery"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
"config/replication/mrf.bin.replay"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
|
||||
@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
|
||||
const MRF_PENDING_CAP: usize = 200_000;
|
||||
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
|
||||
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
|
||||
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
|
||||
/// re-attempts every retained entry, so a target that stays down costs one
|
||||
/// attempt per entry per interval.
|
||||
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
|
||||
}
|
||||
|
||||
/// Acknowledge only the generation read by the recovery leader. Entries appended
|
||||
/// after that generation are retained as a suffix and replayed on the next startup.
|
||||
/// after that generation are retained as a suffix and replayed on the next pass.
|
||||
/// Lock order: recovery leader lock -> MRF journal object lock.
|
||||
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Acquires the MRF recovery leader lock for one journal replay pass.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// On acknowledgement failure the backlog is preserved for the next replay pass and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_millis_now() -> i64 {
|
||||
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Whether a periodic pass may run, given the start time of the last pass on
|
||||
/// any node. A stamp more than one interval ahead of this node's clock is skew
|
||||
/// or a bad value, so it does not suppress replay.
|
||||
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
|
||||
let Some(last_started_ms) = last_started_ms else {
|
||||
return true;
|
||||
};
|
||||
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
|
||||
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
|
||||
}
|
||||
|
||||
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
|
||||
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
|
||||
.await
|
||||
.ok()?;
|
||||
std::str::from_utf8(&data).ok()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
|
||||
if let Err(error) = ReplicationConfigStore::save(
|
||||
storage.clone(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
started_ms.to_string().into_bytes(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to record the MRF replay start time"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One MRF journal replay pass. The recovery leader reads the journal,
|
||||
/// re-drives every entry, and acknowledges the replayed prefix; entries that
|
||||
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
|
||||
///
|
||||
/// Every node runs the periodic loop, and the leader lock only serializes
|
||||
/// passes. With `min_interval`, a pass is skipped when a pass on any node
|
||||
/// started within that interval, read from the shared replay stamp, so a
|
||||
/// cluster replays once per interval rather than once per node. The startup
|
||||
/// pass passes `None` and always runs. A pass that finds entries records its
|
||||
/// start time. Returns whether the journal was read.
|
||||
/// Lock order: recovery leader lock -> replay stamp object lock.
|
||||
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
|
||||
return false;
|
||||
};
|
||||
let started_ms = unix_millis_now();
|
||||
if let Some(interval) = min_interval
|
||||
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(storage).await else {
|
||||
return true;
|
||||
};
|
||||
if !entries.is_empty() {
|
||||
write_mrf_replay_stamp(storage, started_ms).await;
|
||||
}
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
|
||||
/// one pass at a time across the cluster, including the startup pass, and the
|
||||
/// shared replay stamp spaces passes from different nodes by `period`.
|
||||
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(period).await;
|
||||
replay_mrf_journal(&storage, Some(period)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
// Start background tasks
|
||||
pool.start_mrf_persister().await;
|
||||
pool.start_mrf_processor().await;
|
||||
pool.start_mrf_replayer().await;
|
||||
pool.start_force_delete_processor().await;
|
||||
|
||||
pool
|
||||
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Starts the MRF processor — one-shot at startup.
|
||||
///
|
||||
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
|
||||
/// rewrites any entries that could not be admitted for a later startup retry.
|
||||
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
|
||||
/// the pass while the server runs.
|
||||
async fn start_mrf_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
replay_mrf_journal(&storage, None).await;
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
/// Starts the runtime MRF replayer — ongoing background task.
|
||||
///
|
||||
/// Entries reach the journal while the server runs (worker-queue overflow,
|
||||
/// delete-marker purge intents whose watch window drained with targets still
|
||||
/// dirty). Replaying only at startup left them waiting for a restart.
|
||||
async fn start_mrf_replayer(&self) {
|
||||
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
async fn start_force_delete_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
let handle =
|
||||
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
/// writes them to the on-disk MRF file every flush interval (default 10s,
|
||||
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
|
||||
/// entries accumulate. Runtime writers append under the journal lock; only the
|
||||
/// startup recovery leader may remove an acknowledged prefix.
|
||||
/// recovery leader of a replay pass may remove an acknowledged prefix.
|
||||
async fn start_mrf_persister(&self) {
|
||||
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
|
||||
return;
|
||||
@@ -3421,6 +3511,8 @@ mod tests {
|
||||
allow_first_read: Notify,
|
||||
read_count: AtomicUsize,
|
||||
delete_storage_calls: AtomicUsize,
|
||||
delete_error: StdMutex<EcstoreError>,
|
||||
replay_stamp: StdMutex<Option<Vec<u8>>>,
|
||||
reported_size: StdMutex<Option<i64>>,
|
||||
stream_read_bytes: Arc<AtomicUsize>,
|
||||
write_count: AtomicUsize,
|
||||
@@ -3488,6 +3580,24 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let stamp = self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned")
|
||||
.clone()
|
||||
.ok_or(EcstoreError::FileNotFound)?;
|
||||
return Ok(Self::GetObjectReader {
|
||||
object_info: ObjectInfo {
|
||||
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
|
||||
..Default::default()
|
||||
},
|
||||
stream: Box::new(Cursor::new(stamp)),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
}
|
||||
if !object.ends_with("/.replication/resync.bin")
|
||||
&& !object.ends_with("config/replication/mrf.bin")
|
||||
&& !object.ends_with("config/replication/force-delete.bin")
|
||||
@@ -3551,6 +3661,16 @@ mod tests {
|
||||
let lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
|
||||
};
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let mut stamp = Vec::new();
|
||||
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
|
||||
*self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& let Some(replacement) = self
|
||||
.shared
|
||||
@@ -3672,7 +3792,12 @@ mod tests {
|
||||
_opts: Self::ObjectOptions,
|
||||
) -> Result<Self::ObjectInfo, Self::Error> {
|
||||
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(EcstoreError::NotImplemented)
|
||||
Err(self
|
||||
.shared
|
||||
.delete_error
|
||||
.lock()
|
||||
.expect("delete error lock should not be poisoned")
|
||||
.clone())
|
||||
}
|
||||
|
||||
async fn delete_objects(
|
||||
@@ -4171,6 +4296,8 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -4221,6 +4348,103 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_already_purged_source_completes_without_retry() {
|
||||
let arn = "arn:test";
|
||||
for delete_marker_version in [false, true] {
|
||||
for version_id in [Uuid::new_v4(), Uuid::nil()] {
|
||||
for error in [
|
||||
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
EcstoreError::FileVersionNotFound,
|
||||
EcstoreError::FileNotFound,
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: (!delete_marker_version).then_some(version_id),
|
||||
delete_marker_version_id: delete_marker_version.then_some(version_id),
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
replicate_delete_with_outcome(delete, storage).await,
|
||||
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
|
||||
);
|
||||
assert_eq!(
|
||||
shared.delete_storage_calls.load(Ordering::SeqCst),
|
||||
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
|
||||
"the worker must lock and attempt writeback, reading a non-null data version first"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
|
||||
let arn = "arn:test";
|
||||
for (version_purge, target_status, error) in [
|
||||
(
|
||||
true,
|
||||
VersionPurgeStatusType::Pending,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
|
||||
(
|
||||
false,
|
||||
VersionPurgeStatusType::Empty,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: version_purge.then(Uuid::new_v4),
|
||||
delete_marker: !version_purge,
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
targets: if version_purge {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
|
||||
},
|
||||
purge_targets: if version_purge {
|
||||
HashMap::from([(arn.to_string(), target_status)])
|
||||
} else {
|
||||
HashMap::new()
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!replicate_delete_with_outcome(delete, storage).await,
|
||||
"unfinished target work, source I/O failures, and marker creation must remain retryable"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_preserves_persisted_admission_without_live_config() {
|
||||
let arn = "arn:test";
|
||||
@@ -5304,6 +5528,8 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -6268,6 +6494,108 @@ mod tests {
|
||||
assert_eq!(retained.target_arns, entry.target_arns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
|
||||
// A force-delete intent from a superseded generation replays as
|
||||
// Skipped, so a pass that reaches it acknowledges it off the journal.
|
||||
let stale_intent = |object: &str| MrfReplicateEntry {
|
||||
bucket: "mrf-runtime-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let journal_drained = || async {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
};
|
||||
|
||||
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
|
||||
for object in ["first", "second"] {
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
|
||||
assert!(
|
||||
journal_drained().await,
|
||||
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
|
||||
);
|
||||
}
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
|
||||
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
|
||||
let interval = Duration::from_secs(1);
|
||||
let persist_backlog = |object: &str| {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "mrf-node-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[entry]).expect("MRF entry should encode");
|
||||
};
|
||||
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
|
||||
|
||||
persist_backlog("first");
|
||||
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
|
||||
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
|
||||
|
||||
// Node B's timer fires within the same interval with a backlog present.
|
||||
persist_backlog("second");
|
||||
assert!(
|
||||
!replay_mrf_journal(&node_b, Some(interval)).await,
|
||||
"a node must not replay within one interval of another node's pass"
|
||||
);
|
||||
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
|
||||
|
||||
assert!(
|
||||
replay_mrf_journal(&node_b, None).await,
|
||||
"the startup pass should replay regardless of the shared stamp"
|
||||
);
|
||||
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
|
||||
|
||||
persist_backlog("third");
|
||||
tokio::time::sleep(interval).await;
|
||||
assert!(
|
||||
replay_mrf_journal(&node_a, Some(interval)).await,
|
||||
"a periodic pass should replay once the interval has elapsed"
|
||||
);
|
||||
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
|
||||
let interval = Duration::from_secs(300);
|
||||
let now = 1_000_000_000;
|
||||
assert!(mrf_replay_due(None, now, interval));
|
||||
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
|
||||
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
|
||||
assert!(
|
||||
!mrf_replay_due(Some(now + 1_000), now, interval),
|
||||
"small clock skew should still space passes"
|
||||
);
|
||||
assert!(
|
||||
mrf_replay_due(Some(now + 300_000), now, interval),
|
||||
"a stamp an interval ahead must not suppress replay"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_entry_delete_marker_roundtrip() {
|
||||
let dm_vid = Uuid::new_v4();
|
||||
|
||||
@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
// A batch purge or another worker may have removed this version
|
||||
// already. Remote failures still control the final outcome below.
|
||||
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
|
||||
if !source_already_purged {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
}
|
||||
send_event(EventArgs {
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
false
|
||||
source_already_purged
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
|
||||
/// If the marker disappears (deleted before or while the replica landed),
|
||||
/// purge the replicated marker from the targets, retrying failed targets on
|
||||
/// later rounds. When the window drains with targets still dirty, persist the
|
||||
/// purge intent as a durable MRF entry so the next startup replays it through
|
||||
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
|
||||
/// `purge_stale_delete_marker_targets`.
|
||||
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
|
||||
bucket: String,
|
||||
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
|
||||
/// Heal callers retry for the full watch window because the startup MRF
|
||||
/// processor runs before bucket metadata (and thus target clients) finishes
|
||||
/// initializing — the first pass can see `target_client_missing` and a later
|
||||
/// round resolves the client; the replay loop is serial and startup-only, so
|
||||
/// round resolves the client; replay passes are serial background work, so
|
||||
/// blocking it for up to the window per dirty entry is acceptable. Live
|
||||
/// callers run on replication workers where a down target would pin a worker
|
||||
/// for the whole window, so they attempt once and lean on the durable intent
|
||||
|
||||
@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
|
||||
};
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
|
||||
global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
|
||||
use rustfs_utils::get_env_bool;
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
|
||||
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
|
||||
|
||||
@@ -67,15 +67,14 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
|
||||
MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -1638,6 +1637,7 @@ impl RemoteDisk {
|
||||
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
|
||||
if initial_probe_ok {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1709,6 +1709,7 @@ impl RemoteDisk {
|
||||
// Perform basic connectivity check
|
||||
if Self::perform_connectivity_check(&addr).await.is_ok() {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1779,6 +1780,7 @@ impl RemoteDisk {
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
|
||||
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -2010,6 +2012,7 @@ impl RemoteDisk {
|
||||
let operation_result = operation().await;
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2024,6 +2027,7 @@ impl RemoteDisk {
|
||||
// Log success; the waiting guard balances every exit path.
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2453,212 +2457,6 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
/// Prepare a remote token without retrying the RPC. It owns the bucket
|
||||
/// publication fence but does not enter the object metadata queue yet.
|
||||
pub async fn reserve_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"reserve_object_metadata",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.reserve_object_metadata(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
if response.protocol_version != 2 {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
Uuid::from_slice(&response.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or(DiskError::Unexpected)
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token. The caller must cancel it on an indeterminate
|
||||
/// result before allowing any subsequent unprotected mutation.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
|
||||
self.update_object_metadata_reservation(bucket, object, token, timeout, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
|
||||
self.update_object_metadata_reservation(bucket, object, token, timeout, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_object_metadata_reservation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: Duration,
|
||||
renew_only: bool,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"activate_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationActivateRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
renew_only,
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation activation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.activate_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed remote reservation. False cannot prove that an
|
||||
/// earlier claim did not start a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"cancel_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationCancelRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation cancellation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.cancel_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error {
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(response.success)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Publish a staged user-object write using a remote one-shot reservation.
|
||||
/// The caller must retain the bucket lifecycle fence until the tail drains.
|
||||
pub async fn rename_data_with_metadata_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
token: Uuid,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_reserved_data",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let rename = RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
scanner_publication_lease_token: Bytes::new(),
|
||||
bucket_incarnation_id: Bytes::new(),
|
||||
};
|
||||
let mut request = Request::new(ReservedRenameDataRequest {
|
||||
rename: Some(rename),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reserved rename request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.rename_reserved_data(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -5316,63 +5114,6 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let token = Uuid::new_v4();
|
||||
assert!(matches!(
|
||||
remote
|
||||
.reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.renew_object_metadata("bucket", "object", token, Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote.cancel_object_metadata_reservation(token).await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.rename_data_with_metadata_reservation(
|
||||
crate::disk::RUSTFS_META_TMP_BUCKET,
|
||||
"staged",
|
||||
&sample_rename_data_file_info(),
|
||||
"bucket",
|
||||
"object",
|
||||
token,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
|
||||
vec![
|
||||
"/node_service.NodeService/ReserveObjectMetadata",
|
||||
"/node_service.NodeService/ActivateObjectMetadataReservation",
|
||||
"/node_service.NodeService/ActivateObjectMetadataReservation",
|
||||
"/node_service.NodeService/CancelObjectMetadataReservation",
|
||||
"/node_service.NodeService/RenameReservedData",
|
||||
],
|
||||
"reservation operations must never downgrade to legacy methods",
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
|
||||
@@ -445,6 +445,10 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
});
|
||||
// A successful lock RPC over a cached or fresh channel proves the peer is
|
||||
// reachable; reconcile the internode peer-health gauge so it does not stay
|
||||
// stale when no fresh dial fires (issue #9119).
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
|
||||
fn admit_late_release(&self) -> bool {
|
||||
@@ -672,6 +676,7 @@ impl RemoteClient {
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
});
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
|
||||
if let Some(late) = late {
|
||||
late(response).await;
|
||||
@@ -1315,122 +1320,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
// Older nodes refresh normally but return no conversion marker.
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
if !response.success {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
|
||||
Some(":0") => Some(false),
|
||||
Some(":1") => Some(true),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote force_release for {}", lock_id);
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct PhysicalFixture {
|
||||
_directories: Vec<tempfile::TempDir>,
|
||||
@@ -628,3 +629,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
|
||||
let fixture = fixture(0).await;
|
||||
for pool in disks(&fixture.store).await {
|
||||
for disk in pool {
|
||||
let object = object_path(&disk);
|
||||
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
|
||||
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
|
||||
if entry.file_type().await.expect("fixture entry type").is_dir() {
|
||||
fs::remove_dir_all(entry.path())
|
||||
.await
|
||||
.expect("remove task-owned fixture rollback directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
|
||||
let mut all = Vec::new();
|
||||
for pool in &store.pools {
|
||||
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
|
||||
let members = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.expect("identity fixture disk must be online"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut heads = Vec::new();
|
||||
for disk in members {
|
||||
match disk
|
||||
.pool_meta_native_recovery(Request::InspectIdentity {})
|
||||
.await
|
||||
.expect("physical identity inspection")
|
||||
{
|
||||
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
|
||||
_ => panic!("identity inspection must return its physical head"),
|
||||
}
|
||||
}
|
||||
all.push(heads);
|
||||
}
|
||||
all
|
||||
}
|
||||
|
||||
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
|
||||
let fixture = owner_repair_no_backups_fixture().await;
|
||||
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("fixture must provide valid committed pool metadata"),
|
||||
};
|
||||
meta.version = version;
|
||||
let raw = meta
|
||||
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
|
||||
.expect("legacy fixture should encode in its original version");
|
||||
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
|
||||
let all = disks(&fixture.store).await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
for pool in &all {
|
||||
let heads = replicas(&raw, now).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install exact legacy head");
|
||||
}
|
||||
}
|
||||
if stale {
|
||||
let old = replicas(&raw, now - Duration::seconds(10)).await;
|
||||
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
|
||||
.await
|
||||
.expect("install stale same-raw legacy head");
|
||||
}
|
||||
(fixture, raw)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
|
||||
drop(state);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for read in &reads {
|
||||
assert!(
|
||||
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
|
||||
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
|
||||
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
|
||||
let mut source_raw = raw.clone();
|
||||
// A second MessagePack value is readable by legacy decoding but
|
||||
// must not be copied as part of an unverified repair payload.
|
||||
source_raw.push(0xc0);
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
|
||||
"ordinary legacy reads must remain compatible with trailing bytes"
|
||||
);
|
||||
let all_disks = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in all_disks[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install readable legacy source with trailing bytes");
|
||||
}
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await
|
||||
.expect("legacy semantic selection must remain readable");
|
||||
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
drop(state);
|
||||
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"version {version}: no identity shard may change"
|
||||
);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
for target_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
|
||||
_ => panic!("initial V1 metadata must decode"),
|
||||
};
|
||||
meta.version = POOL_META_VERSION;
|
||||
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
|
||||
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
|
||||
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
|
||||
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V1 payload must remain valid"),
|
||||
};
|
||||
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V2 payload must remain valid"),
|
||||
};
|
||||
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
|
||||
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
|
||||
// Pool two follows pool zero; pools zero and one cover both
|
||||
// orderings and both target formats in separate runs.
|
||||
let versions = [first_version, other_version, first_version];
|
||||
let physical = disks(&fixture.store).await;
|
||||
let timestamp = OffsetDateTime::now_utc();
|
||||
for (pool_index, pool) in physical.iter().enumerate() {
|
||||
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
let heads = replicas(raw, timestamp).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
|
||||
}
|
||||
}
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await.expect("equivalent legacy pools should be selectable");
|
||||
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
||||
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
||||
.await.expect("mixed legacy heal must terminate")
|
||||
.expect("equivalent legacy metadata should heal without downgrading the format floor");
|
||||
drop(state);
|
||||
let after = snapshots(&fixture.store).await;
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for (pool_index, read) in reads.iter().enumerate() {
|
||||
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
|
||||
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
assert!(matches!(&read.replica,
|
||||
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
|
||||
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
|
||||
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
|
||||
for (index, snapshot) in after[pool_index].iter().enumerate() {
|
||||
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
|
||||
.await.expect("every physical shard must match its selected legacy payload");
|
||||
}
|
||||
if pool_index != target_pool {
|
||||
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for prepare_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta,
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
|
||||
_ => panic!("baseline V1 metadata must be committed and readable"),
|
||||
};
|
||||
let previous = PoolMetaCommittedCandidate {
|
||||
canonical: legacy_canonical,
|
||||
meta: legacy_meta.clone(),
|
||||
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
|
||||
};
|
||||
let mut next = legacy_meta;
|
||||
next.version = POOL_META_GENERATION_VERSION;
|
||||
next.pools[0].last_update += Duration::seconds(1);
|
||||
let revision = PoolMetaRevision {
|
||||
version: POOL_META_GENERATION_VERSION,
|
||||
cluster_id: fixture.revision.cluster_id,
|
||||
epoch: fixture.revision.epoch,
|
||||
generation: 1,
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
};
|
||||
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
|
||||
.expect("V3 PREPARE with V1 predecessor must encode");
|
||||
assert!(matches!(decode_pool_meta_replica(pending.clone()),
|
||||
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
|
||||
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
let physical = disks(&fixture.store).await;
|
||||
for (index, disk) in physical[prepare_pool].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install task-owned V3 PREPARE evidence");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[prepare_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
|
||||
if raw == &pending && *actual == revision));
|
||||
let other_pool = 1 - prepare_pool;
|
||||
assert!(matches!(&reads[other_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
|
||||
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture
|
||||
.store
|
||||
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("mixed V1/V3 PREPARE heal must terminate");
|
||||
drop(state);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"pool {prepare_pool}: rejection must not mutate identity evidence"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
|
||||
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("current reader must recognize the legacy source"),
|
||||
};
|
||||
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source && revision.version == POOL_META_V1_VERSION));
|
||||
let physical = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install legacy source on pool zero");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve every original head and backup"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
|
||||
"the dangerous source bytes must not be copied into the target pool"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
|
||||
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("safe pool-zero source must decode"),
|
||||
};
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
|
||||
"the dangerous target must remain semantically equivalent to the safe source"
|
||||
);
|
||||
let physical = disks(&fixture.store).await;
|
||||
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[1].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
|
||||
.await
|
||||
.expect("install legacy dangerous target head");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy target heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve all physical heads and backups"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
|
||||
"unsafe target evidence must not be overwritten by the safe source"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
#[cfg(test)]
|
||||
use std::io::Cursor;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
@@ -3720,33 +3719,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
|
||||
}
|
||||
|
||||
/// The storage error of a version copy that kept timing out on a namespace
|
||||
/// lock through its in-place retries, kept typed so the entry failure policy
|
||||
/// defers it.
|
||||
/// A copy whose object namespace lock remained unavailable through its
|
||||
/// in-place retries, retained as a typed deferral.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
struct DecommissionCopyLockTimeout(#[source] Error);
|
||||
struct DecommissionCopyLockUnavailable(#[source] Error);
|
||||
|
||||
fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
|
||||
let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
|
||||
let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
match source {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: lock_bucket,
|
||||
object: lock_object,
|
||||
..
|
||||
} => {
|
||||
data_movement::data_movement_error_is_stage(err, "prepare_put_object")
|
||||
&& lock_bucket == bucket
|
||||
&& lock_object == object
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// A version copy that timed out acquiring a namespace lock leaves the source
|
||||
/// A version copy that could not acquire its namespace lock leaves the source
|
||||
/// in place; the next attempt copies the version again or finds an equivalent
|
||||
/// committed target. The lock is held by another mutation of the key or, when
|
||||
/// this node restarted mid-copy, by its previous incarnation until that lock's
|
||||
/// lease expires on the peers. Either clears by waiting, so the entry is
|
||||
/// deferred instead of being counted as a permanent per-object failure.
|
||||
fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
data_movement::data_movement_context_error(
|
||||
format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockTimeout(err),
|
||||
format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockUnavailable(err),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
|
||||
fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
|
||||
}
|
||||
|
||||
/// The deferral of a version copy whose in-place retries ended on `error`, if
|
||||
@@ -3756,8 +3767,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
|
||||
return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
|
||||
}
|
||||
error
|
||||
.take_if(|err| is_decommission_copy_lock_timeout_error(err))
|
||||
.map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
|
||||
.take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
|
||||
.map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
|
||||
}
|
||||
|
||||
fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
|
||||
@@ -5822,13 +5833,13 @@ enum DecommissionEntryOutcome {
|
||||
/// entry is done and must not be counted as a failure or escalated. A quorum
|
||||
/// failure of the source cleanup is deferred too: the copied source is still
|
||||
/// in place and the next attempt cleans it up again. So is a version copy that
|
||||
/// lost drive quorum or timed out on a namespace lock through its in-place
|
||||
/// lost drive quorum or could not acquire an object namespace lock through its in-place
|
||||
/// retries: the source still holds the version and the next attempt copies it
|
||||
/// again. Every other failure stays terminal.
|
||||
fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
|
||||
if is_decommission_source_cleanup_retryable_error(&err)
|
||||
|| is_decommission_copy_quorum_deferral(&err)
|
||||
|| is_decommission_copy_lock_timeout_deferral(&err)
|
||||
|| is_decommission_copy_lock_unavailable_deferral(&err)
|
||||
{
|
||||
return Ok(DecommissionEntryOutcome::Deferred(err));
|
||||
}
|
||||
@@ -6827,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
|
||||
if data.len() >= 4
|
||||
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
|
||||
// The selection already validates the payload. The runtime-only
|
||||
// flag can only appear in the legacy three-field sequence or map.
|
||||
&& rmp::decode::read_array_len(&mut &data[4..])
|
||||
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
|
||||
.ok()
|
||||
== Some(3)
|
||||
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
|
||||
&& legacy.dont_save
|
||||
{
|
||||
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Ordinary legacy reads retain their compatibility behavior, but bytes that
|
||||
// will be copied to another pool must not contain an unparsed extension.
|
||||
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
|
||||
let payload = data
|
||||
.get(4..)
|
||||
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
|
||||
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
|
||||
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
|
||||
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
|
||||
if decoder.get_ref().position() != payload_len {
|
||||
return Err(Error::other("pool metadata legacy source has trailing bytes"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PoolMetaSelection {
|
||||
meta: PoolMeta,
|
||||
revision: PoolMetaRevision,
|
||||
canonical: Option<Vec<u8>>,
|
||||
// Legacy semantic comparison canonicalizes to V2; a heal must instead
|
||||
// preserve verified raw bytes at the highest observed format floor.
|
||||
legacy_raw: Option<Vec<u8>>,
|
||||
legacy_has_unsafe_flag: bool,
|
||||
replica_state: PoolMetaReplicaState,
|
||||
cas_tokens: Vec<PoolMetaCasToken>,
|
||||
absent: bool,
|
||||
@@ -7222,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: PoolMeta::default(),
|
||||
revision: PoolMetaRevision::legacy(0),
|
||||
canonical: None,
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair: false,
|
||||
repair_write_safe: true,
|
||||
@@ -7276,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: selected.meta.clone(),
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical.clone()),
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -7289,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
|
||||
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
|
||||
// record; divergent backups without it are ambiguous and fail closed.
|
||||
let legacy_has_unsafe_flag = committed
|
||||
.iter()
|
||||
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
|
||||
let mut floor_raw = None;
|
||||
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
|
||||
for (idx, raw, candidate) in committed {
|
||||
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
|
||||
@@ -7302,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
}
|
||||
} else {
|
||||
needs_repair |= selected_raw != &raw;
|
||||
if floor_raw.is_none()
|
||||
&& candidate.revision.version == observed_version
|
||||
&& raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(raw);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
selected = Some((idx, raw, candidate));
|
||||
}
|
||||
}
|
||||
let (_, _, mut selected) =
|
||||
let (_, selected_raw, mut selected) =
|
||||
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
|
||||
if selected.revision.version == observed_version
|
||||
&& selected_raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(selected_raw);
|
||||
}
|
||||
selected.meta.version = observed_version;
|
||||
selected.revision.version = observed_version;
|
||||
Ok(PoolMetaSelection {
|
||||
meta: selected.meta,
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical),
|
||||
legacy_raw: floor_raw,
|
||||
legacy_has_unsafe_flag,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -14534,6 +14606,23 @@ impl ECStore {
|
||||
.canonical
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
|
||||
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
|
||||
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
|
||||
}
|
||||
if selection.legacy_has_unsafe_flag {
|
||||
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
|
||||
}
|
||||
let repair_data = if selection.revision.is_generation_protocol() {
|
||||
canonical.clone()
|
||||
} else {
|
||||
let raw = selection
|
||||
.legacy_raw
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
|
||||
ensure_legacy_pool_meta_payload_complete(&raw)?;
|
||||
raw
|
||||
};
|
||||
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -14544,10 +14633,10 @@ impl ECStore {
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
|
||||
let mut arm = state.arm_transaction();
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
|
||||
.await?;
|
||||
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
|
||||
if confirmed != *canonical {
|
||||
if confirmed != repair_data {
|
||||
return Err(Error::other("pool metadata native heal readback diverged"));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
@@ -28353,7 +28442,7 @@ mod tests {
|
||||
"one version must commit before the next version times out"
|
||||
);
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28387,6 +28476,88 @@ mod tests {
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A target lookup that loses its read-lock quorum must retain the source
|
||||
/// and resume an entry after an earlier version has already committed.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
|
||||
let object = fixture.object;
|
||||
let attempted = Arc::new(AtomicUsize::new(0));
|
||||
let fault = DecommissionTestFaultGuard::install_with_error(
|
||||
Arc::new({
|
||||
let attempted = Arc::clone(&attempted);
|
||||
move |stage, _, faulted, _, success| {
|
||||
stage == "before_target_commit"
|
||||
&& faulted == object
|
||||
&& success
|
||||
&& attempted.fetch_add(1, Ordering::SeqCst) > 0
|
||||
}
|
||||
}),
|
||||
Some(Arc::new(|_, bucket, object| {
|
||||
data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"prepare_put_object",
|
||||
bucket,
|
||||
object,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
)
|
||||
})),
|
||||
);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
fixture.assert_source_retained().await;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.target_versions()
|
||||
.await
|
||||
.expect("first version committed")
|
||||
.versions
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("active decommission")
|
||||
.items_decommission_failed,
|
||||
0
|
||||
);
|
||||
|
||||
drop(fault);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
|
||||
fixture.assert_moved().await;
|
||||
let mut reader = fixture.store.pools[1]
|
||||
.get_object_reader(
|
||||
fixture.bucket,
|
||||
fixture.object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the newer version must remain readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A namespace lock still held on the target key, as by a copy whose node
|
||||
/// was killed mid-write, makes target selection fail with a typed
|
||||
/// object-lock timeout. A real held lock, not an injected error, must defer
|
||||
@@ -28429,7 +28600,7 @@ mod tests {
|
||||
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28838,17 +29009,17 @@ mod tests {
|
||||
timeout(),
|
||||
);
|
||||
for err in [timeout(), prepare] {
|
||||
assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
|
||||
assert!(!is_decommission_copy_quorum_error(&err), "{err}");
|
||||
let mut error = Some(err);
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("a lock timeout copy must be deferred");
|
||||
assert!(error.is_none());
|
||||
assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
|
||||
assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
|
||||
assert!(!is_decommission_copy_quorum_deferral(&deferred));
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
|
||||
.expect("the deferred error must keep its lock-timeout origin");
|
||||
assert!(is_decommission_copy_lock_timeout_error(&failure.0));
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
@@ -28873,6 +29044,43 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
|
||||
let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
let wrap = |stage, err| {
|
||||
data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
|
||||
};
|
||||
let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("matching read-lock quorum must defer");
|
||||
assert!(error.is_none());
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
));
|
||||
for err in [
|
||||
quorum("read", "bucket-a", "object-a"),
|
||||
wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
|
||||
wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
|
||||
wrap(
|
||||
"prepare_put_object",
|
||||
quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
|
||||
),
|
||||
wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
|
||||
wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
|
||||
] {
|
||||
let mut error = Some(err);
|
||||
assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
|
||||
assert!(error.is_some(), "unrelated lock failures must stay terminal");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
@@ -29351,6 +29559,53 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
|
||||
let frame = |payload: Vec<u8>| {
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
data.extend_from_slice(&payload);
|
||||
data
|
||||
};
|
||||
let two_fields = PersistedPoolMetaV1 {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
|
||||
let data = frame(payload.expect("two-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
for dont_save in [false, true] {
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
dont_save,
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
|
||||
let data = frame(payload.expect("three-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
}
|
||||
// This guard is only called for already-decoded replicas; malformed
|
||||
// containers must fail selection rather than becoming repair sources.
|
||||
let truncated = frame(vec![0x93, 0x01]);
|
||||
assert!(matches!(
|
||||
decode_pool_meta_replica(truncated),
|
||||
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_decode_supports_legacy_payload() {
|
||||
let start_time = OffsetDateTime::now_utc();
|
||||
|
||||
@@ -655,19 +655,6 @@ impl Sets {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
self.get_disks_by_key(object)
|
||||
.put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
|
||||
#[derive(Debug)]
|
||||
struct DataMovementStageError {
|
||||
rendered: String,
|
||||
stage: Option<String>,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
@@ -626,6 +627,7 @@ where
|
||||
{
|
||||
Error::other(DataMovementStageError {
|
||||
rendered,
|
||||
stage: None,
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
|
||||
where
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
|
||||
Error::other(DataMovementStageError {
|
||||
rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
|
||||
stage: Some(stage.to_owned()),
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
|
||||
let Error::Io(io_err) = err else {
|
||||
return false;
|
||||
};
|
||||
io_err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<DataMovementStageError>())
|
||||
.is_some_and(|source| source.stage.as_deref() == Some(stage))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -604,69 +604,8 @@ impl LocalDiskWrapper {
|
||||
let super::RenameDataGuards {
|
||||
external_guard,
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
metadata_reservation_token,
|
||||
..
|
||||
} = guards;
|
||||
let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
|
||||
(Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
|
||||
(Some(reservation), None) => Some(reservation),
|
||||
(None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
|
||||
Ok(reservation) => Some(reservation),
|
||||
// A missing token may already have been claimed by an earlier
|
||||
// request that is still publishing; do not claim it was safe.
|
||||
Err(DiskError::DiskOngoingReq) => {
|
||||
return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
|
||||
}
|
||||
Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
},
|
||||
(None, None) => None,
|
||||
};
|
||||
// A queued reservation must acquire its metadata lease before disk
|
||||
// admission. Otherwise a waiting background mutation could hold the
|
||||
// disk permit needed by the earlier lease owner.
|
||||
let metadata_reservation = match metadata_reservation {
|
||||
Some(reservation) => Some(reservation.wait().await),
|
||||
None => None,
|
||||
};
|
||||
// The claimed reservation stays in this cancellable future while disk
|
||||
// admission can wait. Only physical work moves into an owned task.
|
||||
let prechecked_permit = if metadata_reservation.is_some() {
|
||||
// Background admission is exclusive. Waiting for it while holding
|
||||
// this object's lease can deadlock an admitted mutation waiting for
|
||||
// the same lease. An already owned permit does not need to wait.
|
||||
if current_disk_io_context().is_some_and(|context| {
|
||||
context.class == DiskIoClass::Background
|
||||
&& !context
|
||||
.permit
|
||||
.as_ref()
|
||||
.is_some_and(|permit| permit.is_for(&self.io_admission))
|
||||
}) {
|
||||
return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let preflight = self
|
||||
.with_native_resources(async {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.acquire_disk_io_permit().await
|
||||
})
|
||||
.await;
|
||||
match preflight {
|
||||
Ok(Ok(Some(permit))) => Some(permit),
|
||||
Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// The reserved metadata lease must outlive a cancelled RPC waiter.
|
||||
let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
|
||||
Some(Arc::new(external_guard))
|
||||
} else {
|
||||
external_guard
|
||||
};
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -685,54 +624,27 @@ impl LocalDiskWrapper {
|
||||
};
|
||||
let observed = run_owned_mutation(external_guard, move || async move {
|
||||
let mut preflight_rejection = None;
|
||||
let rename = || async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = if let Some(reservation) = metadata_reservation {
|
||||
Box::pin(operation.disk.rename_data_observed_with_reservation(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
(namespace_owner, Some(reservation)),
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await
|
||||
};
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
};
|
||||
let result = if let Some(permit) = prechecked_permit {
|
||||
with_disk_io_permit(Some(permit), async {
|
||||
operation
|
||||
.with_native_resources(operation.track_admitted_disk_operation(
|
||||
"rename_data",
|
||||
move |io_permit| async move {
|
||||
let _io_permit = io_permit;
|
||||
rename().await
|
||||
},
|
||||
Duration::ZERO,
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::Write,
|
||||
true,
|
||||
let result = operation
|
||||
.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await?
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
operation
|
||||
.track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
|
||||
.await
|
||||
};
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
},
|
||||
timeout_duration,
|
||||
)
|
||||
.await;
|
||||
// Health tracking must observe the real disk error, not an Ok tuple.
|
||||
Ok(super::RenameDataObservation {
|
||||
result,
|
||||
@@ -941,6 +853,8 @@ pub(crate) struct DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
|
||||
/// Caller API operations currently executing through the disk health wrapper.
|
||||
api_waiting: AtomicU32,
|
||||
/// Operations rejected because the disk was unavailable for the caller.
|
||||
@@ -968,6 +882,8 @@ impl Default for DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex::new(None),
|
||||
api_waiting: AtomicU32::new(0),
|
||||
total_errors_availability: AtomicU64::new(0),
|
||||
total_errors_timeout: AtomicU64::new(0),
|
||||
@@ -1661,6 +1577,20 @@ impl Default for DiskHealthTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScopedDiskOperationTimeout {
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
previous: Option<Duration>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScopedDiskOperationTimeout {
|
||||
fn drop(&mut self) {
|
||||
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
|
||||
}
|
||||
}
|
||||
|
||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -1686,67 +1616,6 @@ pub struct LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
/// Reserve the metadata mutation lock for one object on this disk instance.
|
||||
/// Unclaimed reservations expire after 60 seconds.
|
||||
pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.reserve_object_metadata(bucket, object, timeout).await
|
||||
}
|
||||
|
||||
/// Register an object metadata reservation in FIFO order on this disk.
|
||||
/// The returned token may be claimed before its physical lease is available.
|
||||
pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.enqueue_object_metadata(bucket, object).await
|
||||
}
|
||||
|
||||
/// Prepare an inert reservation token bound to this disk and object.
|
||||
pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
self.prepare_object_metadata_with_owner(bucket, object, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_object_metadata_with_owner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
|
||||
) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
|
||||
}
|
||||
|
||||
/// Refresh an inert token while the caller is still staging the write.
|
||||
pub async fn renew_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.renew_object_metadata(bucket, object, token)
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token; a cancelled token cannot be reactivated.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
|
||||
pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
|
||||
self.disk.cancel_object_metadata_reservation(token)
|
||||
}
|
||||
|
||||
/// Create a new LocalDiskWrapper
|
||||
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
|
||||
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
|
||||
@@ -2063,6 +1932,23 @@ impl LocalDiskWrapper {
|
||||
self.health_check
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
|
||||
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
|
||||
// The metrics epoch follows this drive across clones and reconnects.
|
||||
// Its scoped override cannot shorten unrelated healthy drives' I/O.
|
||||
let previous = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.replace(timeout);
|
||||
ScopedDiskOperationTimeout {
|
||||
metrics: Arc::clone(&self.metrics),
|
||||
previous,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable health monitoring after disk creation.
|
||||
/// Used to defer health checks until after startup format loading completes.
|
||||
pub fn enable_health_check(&self) {
|
||||
@@ -2730,7 +2616,6 @@ impl LocalDiskWrapper {
|
||||
timeout_duration,
|
||||
timeout_health_action,
|
||||
mutation,
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
@@ -2742,7 +2627,6 @@ impl LocalDiskWrapper {
|
||||
timeout_duration: Duration,
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
identity_prechecked: bool,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
@@ -2764,7 +2648,7 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
// Check if disk is stale
|
||||
if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
|
||||
if let Err(err) = self.check_disk_stale().await {
|
||||
self.metrics.record_availability_error();
|
||||
return Err(err);
|
||||
}
|
||||
@@ -2789,6 +2673,13 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#[cfg(test)]
|
||||
let timeout_duration = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
|
||||
// Execute the operation with timeout
|
||||
let result = tokio::time::timeout(timeout_duration, operation).await;
|
||||
|
||||
@@ -5732,115 +5623,6 @@ mod tests {
|
||||
drop(metadata_lease);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let meta = dir.path().join(RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
|
||||
.await
|
||||
.expect("write format");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
|
||||
disk.make_volume("bucket").await.expect("bucket volume");
|
||||
let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
wrapper.set_disk_id_state(Some(format.erasure.this)).await;
|
||||
disk.format_info.write().await.last_check = None;
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
disk.install_format_probe_hook(false, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv_timeout(Duration::from_secs(30));
|
||||
});
|
||||
let publisher = Arc::clone(&wrapper);
|
||||
let rename = tokio::spawn(async move {
|
||||
publisher
|
||||
.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&FileInfo::default(),
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), entered_rx)
|
||||
.await
|
||||
.expect("disk identity preflight starts")
|
||||
.expect("preflight hook");
|
||||
rename.abort();
|
||||
assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
|
||||
let key = disk.get_object_path("bucket", "object").expect("object path");
|
||||
let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
|
||||
let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
|
||||
release_tx.send(()).expect("release disk identity preflight");
|
||||
if !released_before_io {
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), successor)
|
||||
.await
|
||||
.expect("preflight finishes"),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
released_before_io,
|
||||
"a cancelled request must not retain the reservation before physical mutation starts"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_background_rename_does_not_wait_for_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
let held = wrapper
|
||||
.io_admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("background permit");
|
||||
let fi = FileInfo::default();
|
||||
let rename = crate::disk::with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
wrapper.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&fi,
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
);
|
||||
let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
|
||||
drop(held);
|
||||
let observation = result.expect("reserved mutation must not wait while holding the object lease");
|
||||
assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
|
||||
assert!(observation.rejected_before_publication());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_versions_counts_faulty_drive_availability_rejection() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
@@ -6304,6 +6086,67 @@ mod tests {
|
||||
assert!(health.offline_duration().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
|
||||
let directories = [
|
||||
tempfile::tempdir().expect("faulted drive"),
|
||||
tempfile::tempdir().expect("healthy drive"),
|
||||
];
|
||||
let mut wrappers = Vec::new();
|
||||
for directory in &directories {
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
|
||||
wrappers.push(LocalDiskWrapper::new(disk, false));
|
||||
}
|
||||
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
|
||||
Arc::clone(&wrappers[0].disk),
|
||||
false,
|
||||
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
|
||||
);
|
||||
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
|
||||
tokio::time::pause();
|
||||
let operation = || async {
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
Ok(())
|
||||
};
|
||||
let (faulted, healthy) = tokio::join!(
|
||||
reconnected.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
wrappers[1].track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
);
|
||||
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
|
||||
healthy.expect("healthy drive keeps its caller's deadline");
|
||||
let shorter_started = tokio::time::Instant::now();
|
||||
let shorter = wrappers[0]
|
||||
.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_millis(500),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
|
||||
assert!(shorter_started.elapsed() < Duration::from_secs(1));
|
||||
wrappers[0]
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
|
||||
.await
|
||||
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
|
||||
drop(scope);
|
||||
reconnected
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
|
||||
.await
|
||||
.expect("dropping the scope restores reconnected callers' deadlines");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ignored_timeout_does_not_mark_drive_failure() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
|
||||
FdKey {
|
||||
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
|
||||
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::new();
|
||||
let entry = Arc::new(FdCacheEntry { file, len: 3 });
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file,
|
||||
len: 3,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let old_generation = cache.generation();
|
||||
cache.invalidate_exact("v", "part").await;
|
||||
let current_generation = cache.generation();
|
||||
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
|
||||
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
cache.get(&key("v", "part", direct)).await.is_some(),
|
||||
"new entries must survive the earlier volume predicate"
|
||||
"new entries must survive the earlier volume invalidation"
|
||||
);
|
||||
}
|
||||
cache.clear();
|
||||
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
let replacement = Arc::new(FdCacheEntry {
|
||||
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
|
||||
len: 11,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
for direct in [false, true] {
|
||||
let key = key("v", "part", direct);
|
||||
|
||||
+382
-1265
File diff suppressed because it is too large
Load Diff
@@ -444,29 +444,12 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
#[derive(Debug)]
|
||||
pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
|
||||
impl LocalRenamePreflightRejection {
|
||||
pub(in crate::disk) fn not_dispatched() -> Self {
|
||||
Self(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl RenameDataState {
|
||||
pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
|
||||
Self {
|
||||
metadata_reservation: Some(reservation),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep optional native MPU state machines out of ordinary rename frames.
|
||||
// Construct the future here as well as pinning it, so its temporary stays off
|
||||
// the caller's poll stack. Call only from the existing native MPU branches.
|
||||
@@ -508,15 +491,6 @@ impl LocalDisk {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let mut fi = fi;
|
||||
// Pre-acquired metadata must never reverse quota or multipart-source
|
||||
// lock order. These paths need their own reservation protocol.
|
||||
if state.metadata_reservation.is_some()
|
||||
&& (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
|
||||
{
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
|
||||
Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
@@ -612,27 +586,7 @@ impl LocalDisk {
|
||||
Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
|
||||
None => state.namespace_owner.take(),
|
||||
};
|
||||
// A reserved lease is single-use and bound to the exact disk path.
|
||||
// Keep the independent namespace/source owner alongside it; a pre-held
|
||||
// mutex cannot be acquired again without deadlocking.
|
||||
let reservation_fence = state
|
||||
.metadata_reservation
|
||||
.as_ref()
|
||||
.and_then(|reservation| reservation.fence());
|
||||
let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
|
||||
if multipart_staging.is_some() || quota_fence_claim.is_some() {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let lease = reservation
|
||||
.into_lease_for(&root_destination_object_path, self.reservation_instance)
|
||||
.inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
Arc::new((lease, metadata_owner))
|
||||
} else {
|
||||
os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
|
||||
};
|
||||
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
|
||||
drop(metadata_wait);
|
||||
#[cfg(unix)]
|
||||
let parent_receipt = if native_timing {
|
||||
@@ -646,23 +600,17 @@ impl LocalDisk {
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let mut receipt_mutation = native_timing
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
|
||||
let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&root_destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_owner),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await;
|
||||
drop(namespace_wait);
|
||||
// The volume read lease prevents deletion of this disk's bucket until
|
||||
// publication finishes. Recheck after its wait, before any mutation.
|
||||
if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -2601,23 +2549,8 @@ impl LocalDisk {
|
||||
dst_path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::super::RenameDataObservation {
|
||||
self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed_with_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
|
||||
) -> super::super::RenameDataObservation {
|
||||
let (namespace_owner, metadata_reservation) = owners;
|
||||
let mut state = RenameDataState {
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
..Default::default()
|
||||
};
|
||||
let result = self
|
||||
|
||||
@@ -1002,21 +1002,17 @@ pub(super) fn inspection_with_markers(
|
||||
|
||||
type PreparedDeleteReceiptEntry = (DeleteRecoveryJournal, bool);
|
||||
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<parking_lot::Mutex<HashMap<PathBuf, PreparedDeleteReceiptEntry>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
|
||||
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
|
||||
|
||||
fn put_prepared_delete_receipt(rollback: PathBuf, entry: PreparedDeleteReceiptEntry) {
|
||||
let mut map = PREPARED_DELETE_RECEIPTS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(rollback, entry);
|
||||
static PREPARED_DELETE_RECEIPTS: std::sync::LazyLock<os::ShardedPathCache<PreparedDeleteReceiptEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(PREPARED_DELETE_RECEIPTS_PER_SHARD));
|
||||
|
||||
fn put_prepared_delete_receipt(rollback: impl AsRef<Path>, entry: PreparedDeleteReceiptEntry) {
|
||||
PREPARED_DELETE_RECEIPTS.insert(rollback.as_ref(), entry);
|
||||
}
|
||||
|
||||
fn take_prepared_delete_receipt(rollback: &Path) -> Option<PreparedDeleteReceiptEntry> {
|
||||
PREPARED_DELETE_RECEIPTS.lock().remove(rollback)
|
||||
PREPARED_DELETE_RECEIPTS.remove(rollback)
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
|
||||
@@ -270,21 +270,17 @@ struct ActiveMultipartSourceEntry {
|
||||
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
|
||||
}
|
||||
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<
|
||||
parking_lot::Mutex<std::collections::HashMap<PathBuf, ActiveMultipartSourceEntry>>,
|
||||
> = std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: PathBuf, raw: Bytes, fi: FileInfo) {
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
|
||||
let source = source.as_ref();
|
||||
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
|
||||
return;
|
||||
};
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().cloned()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert(
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.insert(
|
||||
source,
|
||||
ActiveMultipartSourceEntry {
|
||||
raw,
|
||||
@@ -304,56 +300,50 @@ fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> b
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
let (raw, fi, head_stamp) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
(entry.raw.clone(), entry.fi.clone(), entry.head_stamp)
|
||||
};
|
||||
let (raw, fi, head_stamp) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return None;
|
||||
}
|
||||
Some((raw, fi))
|
||||
}
|
||||
|
||||
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get(source).map(|entry| entry.head_stamp) else {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
|
||||
let Some((head_stamp, clean)) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.map(|entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
let Some((head_stamp, clean)) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return false;
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
entry.unsettled_parts.insert(part_number);
|
||||
entry.parts.remove(&part_number);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
|
||||
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.lock()
|
||||
.get(source)
|
||||
.and_then(|entry| entry.fi.data_dir)
|
||||
.with_get(source, |entry| entry.fi.data_dir)
|
||||
.flatten()
|
||||
.filter(|dir| !dir.is_nil())
|
||||
else {
|
||||
return;
|
||||
@@ -362,9 +352,9 @@ pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs
|
||||
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
|
||||
return;
|
||||
};
|
||||
if let Some(entry) = ACTIVE_MULTIPART_SOURCE_HEADS.lock().get_mut(source) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
entry.parts.insert(part.number, (part_stamp, part));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn finish_active_multipart_part_transaction(
|
||||
@@ -373,34 +363,29 @@ pub(super) fn finish_active_multipart_part_transaction(
|
||||
committed: bool,
|
||||
retired_existing_txn: bool,
|
||||
) {
|
||||
let mut map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let Some(part_number) = part_number.filter(|_| committed) else {
|
||||
map.remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return;
|
||||
};
|
||||
if !retired_existing_txn {
|
||||
return;
|
||||
}
|
||||
if let Some(entry) = map.get_mut(source)
|
||||
&& !entry.unsettled_parts.remove(&part_number)
|
||||
{
|
||||
map.remove(source);
|
||||
}
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = {
|
||||
let map = ACTIVE_MULTIPART_SOURCE_HEADS.lock();
|
||||
let entry = map.get(source)?;
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
(entry.head_stamp, data_dir, *expected_stamp, part.clone())
|
||||
};
|
||||
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| {
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
|
||||
})
|
||||
.flatten()?;
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return None;
|
||||
}
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
|
||||
@@ -411,11 +396,11 @@ pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().remove(source);
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.lock().retain(|key, _| !key.starts_with(prefix));
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
|
||||
}
|
||||
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
@@ -1936,6 +1921,7 @@ impl NativeMultipartPublication {
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn prepare_and_install_head_on_thread(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
|
||||
+12
-112
@@ -106,8 +106,6 @@ pub(crate) struct RenameDataGuards {
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
pub(crate) metadata_reservation_token: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
@@ -124,13 +122,6 @@ impl RenameDataObservation {
|
||||
}
|
||||
}
|
||||
|
||||
fn not_dispatched(error: DiskError) -> Self {
|
||||
Self {
|
||||
result: Err(error),
|
||||
preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rejected_before_publication(&self) -> bool {
|
||||
self.result.is_err() && self.preflight_rejection.is_some()
|
||||
}
|
||||
@@ -320,82 +311,6 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Prepare an inert single-use reservation on this disk.
|
||||
/// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
|
||||
pub async fn prepare_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.prepare_object_metadata(bucket, object),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Refresh an inert token while its sender stages the write.
|
||||
pub async fn renew_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.renew_object_metadata(bucket, object, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.renew_object_metadata(bucket, object, token, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token in object mutation order.
|
||||
pub async fn activate_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.activate_object_metadata(bucket, object, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. False does not prove that a claim has
|
||||
/// not already started a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
match self {
|
||||
Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
|
||||
Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Physical ownership belongs to the node hosting the local attachment.
|
||||
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
|
||||
match self {
|
||||
@@ -1336,33 +1251,18 @@ impl Disk {
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => {
|
||||
if guards.metadata_reservation.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
if let Some(token) = guards.metadata_reservation_token {
|
||||
if guards.scanner_publication_lease_token.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
return RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+796
-459
File diff suppressed because it is too large
Load Diff
@@ -429,6 +429,7 @@ impl WritePlan {
|
||||
let checksum = reader.content_hash().clone();
|
||||
let trailer = reader.get_trailer().cloned();
|
||||
|
||||
let delegate_inner_etag = self.compression.is_none() && reader.is_etag_reader();
|
||||
let encrypted = self.encryption.is_some();
|
||||
if let Some(algorithm) = self.compression {
|
||||
reader = HashReader::from_reader(
|
||||
@@ -448,7 +449,8 @@ impl WritePlan {
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
|
||||
.with_inner_etag_delegation(delegate_inner_etag);
|
||||
reader = match encryption.mode {
|
||||
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
|
||||
@@ -160,20 +160,6 @@ pub struct ObjectLockConfigSnapshot {
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
pub(crate) fn owns_lifecycle_guard(&self) -> bool {
|
||||
self._lifecycle_guard.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
|
||||
self.bucket.as_deref() == Some(bucket)
|
||||
&& incarnation.is_some()
|
||||
&& self.bucket_incarnation_id == incarnation
|
||||
&& self
|
||||
._lifecycle_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
|
||||
}
|
||||
|
||||
pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
|
||||
Self {
|
||||
store_id: None,
|
||||
|
||||
@@ -101,12 +101,6 @@ pub(crate) struct NamespaceCommitGuard {
|
||||
counted: bool,
|
||||
}
|
||||
|
||||
impl NamespaceCommitGuard {
|
||||
pub(crate) fn covers_instance(&self, ctx: &Arc<InstanceContext>) -> bool {
|
||||
Arc::ptr_eq(&self.ctx, ctx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for NamespaceCommitGuard {
|
||||
fn drop(&mut self) {
|
||||
if self.counted {
|
||||
@@ -1157,8 +1151,6 @@ mod tests {
|
||||
other.set_scanner_publication_state(false);
|
||||
assert!(first.scanner_publication_state_allowed());
|
||||
let one = first.begin_namespace_commit();
|
||||
assert!(one.covers_instance(&first));
|
||||
assert!(!one.covers_instance(&other));
|
||||
let shared_owner = Arc::clone(&one);
|
||||
let two = first.begin_namespace_commit();
|
||||
assert!(first.namespace_commits_pending());
|
||||
|
||||
@@ -318,6 +318,21 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
|
||||
match rebalance_error_source(err) {
|
||||
Error::Lock(err) => is_rebalance_transient_lock_error(err),
|
||||
// No metadata was written when lock admission failed. Each attempt
|
||||
// must acquire its quorum again before loading or saving this run.
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket,
|
||||
object,
|
||||
..
|
||||
} => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
|
||||
if err.kind() == std::io::ErrorKind::TimedOut {
|
||||
return true;
|
||||
@@ -385,9 +400,7 @@ where
|
||||
match result {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(err) => {
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1)
|
||||
|| !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
|
||||
{
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
@@ -719,6 +732,82 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
|
||||
let mut attempts = 0;
|
||||
retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(if attempts == 1 {
|
||||
resolve_rebalance_meta_save_result(
|
||||
Err(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)),
|
||||
"stop_rebalance",
|
||||
)
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.expect("a later quorum-protected admission must be allowed to succeed");
|
||||
assert_eq!(attempts, 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
|
||||
for max_attempts in [0, 1, 3] {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, max_attempts, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)))
|
||||
})
|
||||
.await;
|
||||
let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
|
||||
assert_eq!(attempts, max_attempts.max(1));
|
||||
assert!(matches!(
|
||||
rebalance_error_source(&error),
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(if attempts == 1 {
|
||||
rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)
|
||||
} else {
|
||||
Error::OperationCanceled
|
||||
}))
|
||||
})
|
||||
.await;
|
||||
assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
|
||||
assert!(matches!(result, Err(Error::OperationCanceled)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
|
||||
for err in [
|
||||
@@ -731,6 +820,20 @@ mod error_source_tests {
|
||||
required: 3,
|
||||
achieved: 2,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: "data-bucket".to_string(),
|
||||
object: REBAL_META_NAME.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
|
||||
object: "other-metadata.bin".to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::other("stale rebalance run rejected: lock acquisition timed out"),
|
||||
Error::other("rebalance distributed run fence lost"),
|
||||
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
|
||||
|
||||
@@ -2538,7 +2538,6 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
|
||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
|
||||
pub(in crate::set_disk) tail_has_metadata_reservations: bool,
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) struct RenameTailCleanup {
|
||||
@@ -3469,8 +3468,6 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: Option<RenameRollbackReceipt>,
|
||||
multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
|
||||
reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
}
|
||||
|
||||
impl<'a> RenameDataFenceOptions<'a> {
|
||||
@@ -3486,8 +3483,6 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: None,
|
||||
multipart_proof_authority: None,
|
||||
multipart_commit_owner: None,
|
||||
reservation_ctx: None,
|
||||
prepared_reservations: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3524,274 +3519,6 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
self.namespace_commit_guard = namespace_commit_guard;
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_reservation_context(
|
||||
mut self,
|
||||
ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
) -> Self {
|
||||
self.reservation_ctx = ctx;
|
||||
self.prepared_reservations = reservations;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct MetadataRenewalTarget {
|
||||
disk: DiskStore,
|
||||
token: Uuid,
|
||||
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
|
||||
}
|
||||
|
||||
struct MetadataRenewalGroup {
|
||||
targets: Arc<parking_lot::Mutex<HashMap<usize, MetadataRenewalTarget>>>,
|
||||
task: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl Drop for MetadataRenewalGroup {
|
||||
fn drop(&mut self) {
|
||||
self.task.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameMetadataReservation {
|
||||
disk: DiskStore,
|
||||
token: Option<Uuid>,
|
||||
renewed_at: Arc<parking_lot::Mutex<tokio::time::Instant>>,
|
||||
renewal: Option<(Arc<MetadataRenewalGroup>, usize)>,
|
||||
}
|
||||
|
||||
impl RenameMetadataReservation {
|
||||
fn stop_renewal(&mut self) {
|
||||
if let Some((group, index)) = self.renewal.take() {
|
||||
let mut targets = group.targets.lock();
|
||||
targets.remove(&index);
|
||||
if targets.is_empty() {
|
||||
group.task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn cancel(&mut self) -> disk::error::Result<()> {
|
||||
self.stop_renewal();
|
||||
let Some(token) = self.token else { return Ok(()) };
|
||||
match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
|
||||
Ok(Ok(true)) => {
|
||||
self.token = None;
|
||||
Ok(())
|
||||
}
|
||||
Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) => Err(error),
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameMetadataReservation {
|
||||
fn drop(&mut self) {
|
||||
self.stop_renewal();
|
||||
let Some(token) = self.token.take() else { return };
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
let disk = self.disk.clone();
|
||||
runtime.spawn(async move {
|
||||
let _ = disk.cancel_object_metadata_reservation(token).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_rename_metadata_reservations(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
owns_bucket_fence: bool,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
let prepared: Vec<disk::error::Result<Option<RenameMetadataReservation>>> = join_all(disks.iter().map(|disk| async move {
|
||||
let Some(disk) = disk else { return Ok(None) };
|
||||
if disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, disk, None, false).await?;
|
||||
}
|
||||
let prepared_at = tokio::time::Instant::now();
|
||||
let token = if disk.is_local() && owns_bucket_fence {
|
||||
let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
|
||||
.await
|
||||
.ok_or(DiskError::DiskOngoingReq)?;
|
||||
store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
|
||||
.await?
|
||||
};
|
||||
// No await separates receipt from installing the cancellation owner.
|
||||
Ok(Some(RenameMetadataReservation {
|
||||
disk: disk.clone(),
|
||||
token: Some(token),
|
||||
renewed_at: Arc::new(parking_lot::Mutex::new(prepared_at)),
|
||||
renewal: None,
|
||||
}))
|
||||
}))
|
||||
.await;
|
||||
let mut reservations = Vec::with_capacity(prepared.len());
|
||||
let mut failed = false;
|
||||
for result in prepared {
|
||||
match result {
|
||||
Ok(reservation) => reservations.push(reservation),
|
||||
Err(_) => {
|
||||
failed = true;
|
||||
reservations.push(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
// Prepared tokens are inert; dropping them cannot leave a queued writer.
|
||||
return Ok(None);
|
||||
}
|
||||
let targets = Arc::new(parking_lot::Mutex::new(HashMap::new()));
|
||||
for (index, reservation) in reservations.iter().enumerate() {
|
||||
if let Some(reservation) = reservation {
|
||||
let token = reservation.token.ok_or(DiskError::Unexpected)?;
|
||||
targets.lock().insert(
|
||||
index,
|
||||
MetadataRenewalTarget {
|
||||
disk: reservation.disk.clone(),
|
||||
token,
|
||||
renewed_at: Arc::clone(&reservation.renewed_at),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
if !targets.lock().is_empty() {
|
||||
let renewal_targets = Arc::clone(&targets);
|
||||
let renewal_ctx = Arc::clone(ctx);
|
||||
let renewal_bucket = bucket.to_owned();
|
||||
let renewal_object = object.to_owned();
|
||||
// One task renews the whole fanout; short writes create no renewal RPCs.
|
||||
let task = tokio::spawn(async move {
|
||||
let mut delay = Duration::from_secs(20);
|
||||
loop {
|
||||
tokio::time::sleep(delay).await;
|
||||
let current: Vec<_> = renewal_targets.lock().values().cloned().collect();
|
||||
if current.is_empty() {
|
||||
break;
|
||||
}
|
||||
// Once any acknowledgment is too old, the complete fanout can
|
||||
// no longer be safely activated. Stop retrying stale peers;
|
||||
// their receiver-side tokens still expire on their own.
|
||||
if current
|
||||
.iter()
|
||||
.any(|target| target.renewed_at.lock().elapsed() >= Duration::from_secs(60))
|
||||
{
|
||||
break;
|
||||
}
|
||||
let results = join_all(current.into_iter().map(|target| {
|
||||
let ctx = Arc::clone(&renewal_ctx);
|
||||
let bucket = &renewal_bucket;
|
||||
let object = &renewal_object;
|
||||
async move {
|
||||
let started = tokio::time::Instant::now();
|
||||
let admitted = !target.disk.is_local()
|
||||
|| crate::store::admit_local_disk(&ctx, &target.disk, None, false).await.is_ok();
|
||||
if admitted
|
||||
&& target
|
||||
.disk
|
||||
.renew_object_metadata(bucket, object, target.token, Duration::from_secs(5))
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
*target.renewed_at.lock() = started;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
delay = if results.into_iter().all(|success| success) {
|
||||
Duration::from_secs(20)
|
||||
} else {
|
||||
Duration::from_secs(5)
|
||||
};
|
||||
}
|
||||
});
|
||||
let group = Arc::new(MetadataRenewalGroup { targets, task });
|
||||
for (index, reservation) in reservations.iter_mut().enumerate() {
|
||||
if let Some(reservation) = reservation {
|
||||
reservation.renewal = Some((Arc::clone(&group), index));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
pub(crate) fn align_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> Option<Vec<Option<RenameMetadataReservation>>> {
|
||||
if reservations.len() != disks.len() {
|
||||
return None;
|
||||
}
|
||||
let mut aligned = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let Some(disk) = disk else {
|
||||
aligned.push(None);
|
||||
continue;
|
||||
};
|
||||
let index = reservations
|
||||
.iter()
|
||||
.position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
|
||||
aligned.push(reservations[index].take());
|
||||
}
|
||||
// Disks lost while staging need no activation; dropping their still-inert
|
||||
// reservations schedules cancellation without blocking the commit.
|
||||
Some(aligned)
|
||||
}
|
||||
|
||||
async fn activate_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
// Freeze renewal before activation. A failed refresh keeps the last
|
||||
// acknowledged timestamp, so stale tokens fall back before dispatch.
|
||||
for reservation in reservations.iter_mut().flatten() {
|
||||
reservation.stop_renewal();
|
||||
}
|
||||
if reservations
|
||||
.iter()
|
||||
.flatten()
|
||||
.any(|reservation| reservation.renewed_at.lock().elapsed() >= Duration::from_secs(50))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
|
||||
if reservation.disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
|
||||
}
|
||||
let token = reservation.token.ok_or(DiskError::Unexpected)?;
|
||||
reservation
|
||||
.disk
|
||||
.activate_object_metadata(bucket, object, token, Duration::from_secs(5))
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
if activated.into_iter().any(|result| result.is_err()) {
|
||||
let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
|
||||
if cancelled.into_iter().any(|result| result.is_err()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
@@ -3803,7 +3530,7 @@ type RenameDataLegacyTuple = (
|
||||
Option<OldCurrentSize>,
|
||||
);
|
||||
|
||||
pub(crate) fn put_rename_early_ack_enabled() -> bool {
|
||||
fn put_rename_early_ack_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
|
||||
}
|
||||
|
||||
@@ -4169,7 +3896,6 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4266,8 +3992,6 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority: _,
|
||||
multipart_commit_owner: _,
|
||||
reservation_ctx,
|
||||
prepared_reservations,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -4283,11 +4007,6 @@ impl SetDisks {
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
|
||||
(Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
|
||||
(_, _) => None,
|
||||
};
|
||||
let tail_has_metadata_reservations = reservations.is_some();
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let coordinator_disks = fanout_disks.clone();
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
@@ -4320,11 +4039,6 @@ impl SetDisks {
|
||||
let file_info = file_info.clone();
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let reservation = reservations
|
||||
.as_mut()
|
||||
.and_then(|entries| entries.get_mut(i))
|
||||
.and_then(Option::take);
|
||||
let reservation_ctx = reservation_ctx.clone();
|
||||
tasks.spawn(async move {
|
||||
let _namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
@@ -4335,17 +4049,6 @@ impl SetDisks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let disk_admission = if reservation.is_some() && disk.is_local() {
|
||||
let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
|
||||
// The fanout owner already covers this instance through
|
||||
// physical completion; only add a disk owner when it does not.
|
||||
let covered = namespace_commit_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.covers_instance(ctx));
|
||||
crate::store::admit_local_disk(ctx, &disk, None, !covered).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let disk_wait_location_stage = if disk.is_local() {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
|
||||
} else {
|
||||
@@ -4371,11 +4074,6 @@ impl SetDisks {
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
dispatch_state = RenameDispatchState::MayHavePublished;
|
||||
let _foreground_active_guard = rename_foreground_active_guard(i);
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
|
||||
Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
|
||||
} else {
|
||||
namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
|
||||
};
|
||||
let observed = disk
|
||||
.rename_data_borrowed_with_fence_observed(
|
||||
&src_bucket,
|
||||
@@ -4385,8 +4083,9 @@ impl SetDisks {
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
|
||||
namespace_owner,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4664,7 +4363,6 @@ impl SetDisks {
|
||||
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
|
||||
commit.map(|mut commit| {
|
||||
commit.tail_drain = Some(tail_drain);
|
||||
commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
|
||||
attach_tail_owner_started,
|
||||
@@ -4734,8 +4432,6 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority,
|
||||
multipart_commit_owner,
|
||||
reservation_ctx: _,
|
||||
prepared_reservations: _,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -5205,7 +4901,6 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7924,45 +7619,6 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[tokio::test]
|
||||
async fn renewal_stops_after_the_disk_loses_admission_and_token_expires() {
|
||||
let (_dir, disk) = read_multiple_test_disk("bucket", &[]).await;
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(endpoint.clone(), Some(disk.clone()));
|
||||
let reservations = prepare_rename_metadata_reservations(&[Some(disk)], "bucket", "object", Uuid::new_v4(), &ctx, false)
|
||||
.await
|
||||
.expect("prepare")
|
||||
.expect("complete fanout");
|
||||
let group = Arc::clone(
|
||||
&reservations[0]
|
||||
.as_ref()
|
||||
.expect("reservation")
|
||||
.renewal
|
||||
.as_ref()
|
||||
.expect("renewal")
|
||||
.0,
|
||||
);
|
||||
ctx.local_disk_map().write().await.remove(&endpoint);
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(21)).await;
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::advance(Duration::from_secs(41)).await;
|
||||
for _ in 0..10 {
|
||||
if group.task.is_finished() {
|
||||
break;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
tokio::time::resume();
|
||||
assert!(group.task.is_finished(), "stale tokens must not cause indefinite renewal attempts");
|
||||
drop(reservations);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn commit_fault_schedule_keeps_multiple_disk_phases_independent() {
|
||||
use rename_fanout_barrier::{PHASE_CLEANUP, PHASE_RENAME, arm, checkpoint};
|
||||
|
||||
@@ -161,7 +161,7 @@ impl SetDisks {
|
||||
)
|
||||
});
|
||||
let mut metadata = FileMeta::new();
|
||||
for parts in versions.into_values() {
|
||||
for (version_id, parts) in versions {
|
||||
let errors = parts
|
||||
.iter()
|
||||
.zip(&missing)
|
||||
@@ -173,15 +173,25 @@ impl SetDisks {
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
|
||||
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((read, write)) => (
|
||||
usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
|
||||
usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
|
||||
),
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
|
||||
Err(DiskError::ErasureReadQuorum) if complete => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
|
||||
let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
|
||||
// A null-version system record must carry the same commit proof
|
||||
// as GET. Listing only needs the selected metadata, so no disk
|
||||
// handles are retained from the shared read selector.
|
||||
Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
|
||||
} else {
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
|
||||
};
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
|
||||
@@ -410,10 +410,6 @@ impl ObjectLockDiagGuard {
|
||||
self
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
self.guard.downgrade_committed_tail().await
|
||||
}
|
||||
|
||||
/// Whether the underlying namespace lock's heartbeat has observed a
|
||||
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
|
||||
/// point on this so a stale lock holder does not race a double-write.
|
||||
@@ -906,10 +902,6 @@ mod core;
|
||||
pub(crate) use core::io_primitives::disk_call_counters;
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
pub(crate) use core::io_primitives::{
|
||||
RenameMetadataReservation, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
|
||||
put_rename_early_ack_enabled,
|
||||
};
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
@@ -4704,6 +4696,17 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
|
||||
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
|
||||
}
|
||||
|
||||
// Reusing a held writer additionally requires the same quorum, so repeated
|
||||
// clients must not give different weight to an otherwise identical domain.
|
||||
fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
|
||||
!left.is_empty()
|
||||
&& left.len() == right.len()
|
||||
&& left.iter().all(|client| {
|
||||
left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
&& right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
})
|
||||
}
|
||||
|
||||
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
@@ -5653,9 +5656,9 @@ impl SetDisks {
|
||||
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
|
||||
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
|
||||
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
|
||||
(true, true) => {
|
||||
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
|
||||
}
|
||||
// Distributed lock clients receive the raw ObjectKey; the pool
|
||||
// namespace is not part of the key stored by their backends.
|
||||
(true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -9046,6 +9049,13 @@ mod tests {
|
||||
"constructing an object lock must not clone each client Arc"
|
||||
);
|
||||
|
||||
// Check shared ownership before acquisition can spawn pending cleanup.
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
|
||||
let write_error = write_lock
|
||||
.get_write_lock(Duration::from_millis(500))
|
||||
.await
|
||||
@@ -9060,11 +9070,6 @@ mod tests {
|
||||
),
|
||||
"the shared client representation must preserve the exact write quorum result: {write_error}"
|
||||
);
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
let read_guard = read_lock
|
||||
.get_read_lock(Duration::from_millis(500))
|
||||
.await
|
||||
@@ -9888,7 +9893,7 @@ mod tests {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn namespace_lock_domain_includes_pool_namespace() {
|
||||
async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
@@ -9927,11 +9932,25 @@ mod tests {
|
||||
"sets in the same pool share the object namespace lock domain"
|
||||
);
|
||||
assert!(
|
||||
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pool namespaces must not be deduplicated solely by identical clients"
|
||||
same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pools using identical backends lock the same raw object key"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
|
||||
let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
|
||||
assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
|
||||
assert!(!same_distributed_lock_quorum(
|
||||
&[a.clone(), a.clone(), b.clone()],
|
||||
&[a.clone(), b.clone(), b]
|
||||
));
|
||||
assert!(!same_distributed_lock_quorum(&[], &[]));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn streaming_reader_holds_read_lock_until_eof() {
|
||||
|
||||
@@ -34,7 +34,6 @@ mod heal_lock_wait_tests;
|
||||
#[derive(Clone, Copy)]
|
||||
struct ExplicitVersionHeal<'a> {
|
||||
opts: &'a HealOpts,
|
||||
staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
|
||||
allow_regeneration: bool,
|
||||
preserve_unrepairable: bool,
|
||||
commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
|
||||
@@ -42,58 +41,6 @@ struct ExplicitVersionHeal<'a> {
|
||||
retirement: Option<&'a MarkerRetirementContext<'a>>,
|
||||
}
|
||||
|
||||
async fn restore_heal_exclusive_guard(
|
||||
guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> disk::error::Result<()> {
|
||||
tokio::time::timeout(get_lock_acquire_timeout(), async {
|
||||
loop {
|
||||
{
|
||||
let mut guard = guard.lock().await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
// A writer may have waited on a peer that upgraded while
|
||||
// another peer was still waiting for its readers to drain.
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
return if matches!(waiting, Ok(Some(false))) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::Timeout)
|
||||
};
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
let waiting =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
|
||||
drop(tokio::spawn(async move {
|
||||
let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}));
|
||||
}
|
||||
|
||||
// Keep the large heal state machine and its construction temporary out of
|
||||
// each dispatch caller's poll frame. Ordinary object writes do not use this
|
||||
// heal-only allocation boundary.
|
||||
@@ -897,7 +844,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: None,
|
||||
@@ -1048,7 +994,6 @@ impl SetDisks {
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
let ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard,
|
||||
allow_regeneration: allow_explicit_version_regen,
|
||||
preserve_unrepairable,
|
||||
commit_fence,
|
||||
@@ -1756,29 +1701,7 @@ impl SetDisks {
|
||||
let dst_data_dir = data_dir;
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
let mut shared_staging = false;
|
||||
if let Some(guard) = staging_guard {
|
||||
let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.supports_heal_lock_transition().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if supported {
|
||||
let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.downgrade_committed_tail().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if converted {
|
||||
shared_staging = true;
|
||||
} else {
|
||||
// Some peers may have converted. Abort without committing;
|
||||
// dropping the guard releases each retained lease.
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
let stage = async {
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = async {
|
||||
let erasure_info = latest_meta.erasure.clone();
|
||||
let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
|
||||
|
||||
@@ -2003,43 +1926,17 @@ impl SetDisks {
|
||||
|
||||
writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
|
||||
Ok(None)
|
||||
};
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
|
||||
if let Some(guard) = staging_guard {
|
||||
tokio::select! {
|
||||
result = stage => result,
|
||||
_ = wait_for_heal_writer(guard) => {
|
||||
// The UUID prefix is private to this attempt. Cleanup must not
|
||||
// retain the namespace lock while a foreground writer waits;
|
||||
// the stale-tmp sweeper also covers a cancelled disk operation.
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
};
|
||||
}
|
||||
.await;
|
||||
match stage_result {
|
||||
Ok(None) => {}
|
||||
failure => {
|
||||
if shared_staging {
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
} else {
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}
|
||||
// Release readers and writers before removing all parts
|
||||
// staged under this heal's exclusively owned prefix.
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
return failure.map(|error| (result, error));
|
||||
}
|
||||
}
|
||||
if shared_staging
|
||||
&& let Some(guard) = staging_guard
|
||||
&& let Err(error) = restore_heal_exclusive_guard(guard).await
|
||||
{
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
|
||||
if _read_repair_read_lock_guard
|
||||
@@ -3719,7 +3616,6 @@ impl SetDisks {
|
||||
&version,
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(can_commit),
|
||||
@@ -3780,7 +3676,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &read_opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -3807,7 +3702,7 @@ impl SetDisks {
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
|
||||
Some(Arc::new(tokio::sync::Mutex::new(
|
||||
Some(Arc::new(
|
||||
measure_heal_object_lock(
|
||||
"heal_object",
|
||||
HealObjectLockKind::Write,
|
||||
@@ -3819,7 +3714,7 @@ impl SetDisks {
|
||||
.narrow_to_disk()
|
||||
.unwrap_or_else(DiskError::other)
|
||||
})?,
|
||||
)))
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -3837,18 +3732,10 @@ impl SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let write_lock_loss = if let Some(guard) = &_write_lock_guard {
|
||||
guard.lock().await.lock_lost_signal()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let locks_lost = || {
|
||||
write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
|
||||
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
};
|
||||
// Recovery cleanup can outlive its future. Never permit reads until
|
||||
// every physical operation holding its recovery owner has finished.
|
||||
let recovery_tail = Arc::new(());
|
||||
let mut recovery_removed_version = false;
|
||||
if !opts.read_repair
|
||||
&& _write_lock_guard.is_some()
|
||||
@@ -3875,7 +3762,6 @@ impl SetDisks {
|
||||
Arc::new((
|
||||
scope.clone(),
|
||||
self.ctx.begin_namespace_commit(),
|
||||
Arc::clone(&recovery_tail),
|
||||
_write_lock_guard.clone(),
|
||||
transaction_guard.clone(),
|
||||
)) as Arc<dyn Send + Sync>
|
||||
@@ -3999,9 +3885,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -4026,9 +3909,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -6167,7 +6047,6 @@ mod heal_result_report_tests {
|
||||
"",
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -10758,369 +10637,6 @@ mod heal_result_report_tests {
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-storage-entry";
|
||||
let object = "object.bin";
|
||||
let expected = vec![0x37; 1024 * 1024];
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external object data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&missing).await.expect("remove shard");
|
||||
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
let result = heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
read_repair: false,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
(result, absence)
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("read-repair reached shard reconstruction");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_millis(200),
|
||||
set.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("background heal reconstruction should allow concurrent readers")
|
||||
.expect("open object while reconstruction is paused");
|
||||
let mut during_heal = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
|
||||
.await
|
||||
.expect("read completes while reconstruction is paused")
|
||||
.expect("read object during reconstruction");
|
||||
assert_eq!(during_heal, expected);
|
||||
drop(reader);
|
||||
release.notify_one();
|
||||
let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("repair finishes")
|
||||
.expect("repair task joins");
|
||||
let (_, error) = result.expect("repair operation succeeds");
|
||||
assert!(error.is_none(), "repair failed: {error:?}");
|
||||
assert!(absence.is_none());
|
||||
assert!(missing.exists(), "missing shard was not restored");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read repaired object");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-long-put";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let put = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes after releasing reconstruction")
|
||||
.expect("heal task joins");
|
||||
put.expect("foreground PUT finishes during reconstruction")
|
||||
.expect("foreground PUT must not time out");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
drop(_pause);
|
||||
let mut absence = None;
|
||||
let (_, retry_error) = set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a later heal must retry the current object");
|
||||
assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read foreground write after heal");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-version-delete";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create versioned bucket");
|
||||
let original = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original version");
|
||||
let version = original.version_id.expect("version id").to_string();
|
||||
let survivor = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x52; 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write surviving version")
|
||||
.version_id
|
||||
.expect("surviving version id");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, &version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("read original metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal_version = version.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
&heal_version,
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut None,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let delete = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes")
|
||||
.expect("heal task joins");
|
||||
delete
|
||||
.expect("foreground delete finishes")
|
||||
.expect("foreground delete succeeds");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
let result = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
|
||||
"stale heal must not resurrect the deleted version: {result:?}"
|
||||
);
|
||||
let mut survivor = set
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(survivor.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("surviving version remains readable");
|
||||
let mut body = Vec::new();
|
||||
survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
|
||||
assert_eq!(body, vec![0x52; 1024]);
|
||||
for disk in &disks {
|
||||
let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
|
||||
assert!(
|
||||
matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
|
||||
"deleted version must remain absent on each disk: {physical:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
|
||||
let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
|
||||
@@ -978,7 +978,7 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
if !root_prefix.is_empty() {
|
||||
if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
|
||||
let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -1003,6 +1003,13 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.into_iter()
|
||||
.map(|sha_dir| sha_dir.trim_end_matches('/').to_owned())
|
||||
.filter(|sha_dir| sha_dir.len() == 64 && sha_dir.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
.map(|sha_dir| {
|
||||
if root_prefix.is_empty() {
|
||||
sha_dir
|
||||
} else {
|
||||
format!("{root_prefix}/{sha_dir}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let listed_upload_dirs = stream::iter(sha_dirs)
|
||||
.map(|sha_dir| {
|
||||
@@ -1125,6 +1132,7 @@ impl SetDisks {
|
||||
orig_bucket: &str,
|
||||
error_path: &str,
|
||||
root_prefix: &str,
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.is_empty() {
|
||||
@@ -1166,7 +1174,8 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
|
||||
let required_discoveries = if require_all_disks { disks.len() } else { discovery_quorum };
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, required_discoveries) {
|
||||
return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
|
||||
}
|
||||
|
||||
@@ -1188,7 +1197,7 @@ impl SetDisks {
|
||||
|
||||
pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
|
||||
let (_, paths, _, _) = self
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "", false)
|
||||
.await?;
|
||||
Ok(paths.into_iter().next())
|
||||
}
|
||||
@@ -1201,8 +1210,9 @@ impl SetDisks {
|
||||
upload_identity: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
|
||||
let (_, candidate_paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
|
||||
let (_, candidate_paths, _, has_minority_candidate) = self
|
||||
.discover_multipart_upload_paths(bucket, object, &expected_parent, false)
|
||||
.await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::DecommissionCapacityBlocked {
|
||||
message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks"
|
||||
@@ -1751,6 +1761,54 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
// Every disk must confirm absence in both namespaces before cleanup can
|
||||
// remove catalog protection from the bucket.
|
||||
for root_prefix in ["", DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||
let (disks, paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, "", root_prefix, true).await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
for upload_path in paths {
|
||||
let (sha_dir, _) = upload_path
|
||||
.rsplit_once('/')
|
||||
.filter(|(directory, upload_id)| !directory.is_empty() && !upload_id.is_empty())
|
||||
.ok_or(DiskError::CorruptedFormat)?;
|
||||
let (metadata, errors) =
|
||||
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, "", false, false, false)
|
||||
.await?;
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
if let Some(error) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(to_object_err(error.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
let (_, modified, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum);
|
||||
let file = Self::pick_valid_fileinfo(&metadata, modified, etag, read_quorum)?;
|
||||
let (Some(owner_bucket), Some(owner_object)) = (
|
||||
file.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
||||
file.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
|
||||
) else {
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
};
|
||||
if owner_bucket.is_empty()
|
||||
|| owner_object.is_empty()
|
||||
|| Self::get_multipart_sha_dir(owner_bucket, owner_object)
|
||||
!= sha_dir
|
||||
.strip_prefix(DATA_MOVEMENT_MULTIPART_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.unwrap_or(sha_dir)
|
||||
{
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
if owner_bucket == bucket {
|
||||
return Err(Error::BucketNotEmpty(bucket.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||
&self,
|
||||
@@ -1762,7 +1820,8 @@ impl SetDisks {
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
|
||||
let (disks, candidate_paths, discovery_quorum, _) =
|
||||
self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
|
||||
let listed_uploads = stream::iter(candidate_paths)
|
||||
.map(|upload_path| {
|
||||
let disks = &disks;
|
||||
@@ -2373,7 +2432,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let multipart_part_size = data.size();
|
||||
let stream = mem::replace(
|
||||
&mut data.stream,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
|
||||
);
|
||||
|
||||
let write_path = classify_multipart_part_write_path(multipart_part_size, fi.erasure.block_size);
|
||||
@@ -10001,7 +10060,7 @@ mod tests {
|
||||
|
||||
// Real disk callbacks and health probes must share wall-clock deadlines.
|
||||
// Advancing Tokio time can expire unrelated healthy-drive probes.
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("2"))], async {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, None::<&str>)], async {
|
||||
let (dirs, store) = crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks().await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "mpu-quorum-success-physical";
|
||||
@@ -10028,6 +10087,8 @@ mod tests {
|
||||
local.observe_multipart_cleanup_for_test(&upload_path)
|
||||
}).collect::<Vec<_>>();
|
||||
let disk::Disk::Local(paused_local) = disks[3].as_ref() else { panic!("actual local physical tail") };
|
||||
// Fault only the held leg; healthy disks keep their normal I/O budgets.
|
||||
let _paused_timeout = paused_local.override_operation_timeout_for_test(Duration::from_secs(2));
|
||||
let io_target = paused_local.get_disk().get_object_path_for_io(bucket, object).expect("real IO target");
|
||||
let mutation_key = paused_local.get_disk().get_object_path(bucket, object).expect("logical native mutation key");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
@@ -10184,7 +10245,15 @@ mod tests {
|
||||
let size = copy_info.get_actual_size().expect("actual source size");
|
||||
copy_info.put_object_reader = Some(PutObjReader::new(HashReader::from_stream(reader.stream, size, size, None, None, false).expect("actual Copy body stream")));
|
||||
tokio::time::timeout(Duration::from_secs(5), store.copy_object(bucket, object, bucket, copy_object, &mut copy_info, &ObjectOptions::default(), &ObjectOptions::default()))
|
||||
.await.expect("cross-key Copy cannot wait on the old successful logical owner").expect("actual CopyObject");
|
||||
.await.expect("cross-key Copy cannot wait on the old successful logical owner")
|
||||
.unwrap_or_else(|error| {
|
||||
let health = disks.iter().enumerate().map(|(slot, disk)| {
|
||||
let disk::Disk::Local(local) = disk.as_ref() else { panic!("actual local Copy backend") };
|
||||
let metrics = local.metrics_snapshot();
|
||||
(slot, disk.runtime_state(), metrics.total_errors_timeout, metrics.total_errors_availability, metrics.total_waiting, metrics.api_calls)
|
||||
}).collect::<Vec<_>>();
|
||||
panic!("actual CopyObject: {error:?}; drive health (slot, state, timeouts, unavailable, waiting, calls): {health:?}");
|
||||
});
|
||||
drop(copy_info);
|
||||
let mut copied = store.get_object_reader(bucket, copy_object, None, HeaderMap::new(), &ObjectOptions::default()).await.expect("actual copied body");
|
||||
let mut actual = Vec::new();
|
||||
@@ -11213,6 +11282,163 @@ mod tests {
|
||||
assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_distinguishes_bucket_ownership() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-target";
|
||||
let other = "multipart-cleanup-other";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create target bucket");
|
||||
disk.make_volume(other).await.expect("create other bucket");
|
||||
}
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an empty upload namespace permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(other, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in other bucket");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an upload owned by another bucket permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in target bucket");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_requires_every_disk_online() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-offline";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks.disks.write().await[3] = None;
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing preserves its discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("an offline disk must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the offline disk must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
set_disks.disks.write().await[3] = Some(disk_stores[3].clone());
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_a_failed_disk_enumeration() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-list-error";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let sha_dir = SetDisks::get_multipart_sha_dir(bucket, "object.bin");
|
||||
let multipart_dir = temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET);
|
||||
tokio::fs::create_dir_all(&multipart_dir)
|
||||
.await
|
||||
.expect("create multipart staging directory");
|
||||
tokio::fs::write(multipart_dir.join(&sha_dir), b"not a directory")
|
||||
.await
|
||||
.expect("replace one upload parent directory with a file");
|
||||
multipart_upload_paths_on_disk(disk_stores[0].clone(), bucket, "")
|
||||
.await
|
||||
.expect_err("the affected disk must fail directory enumeration");
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing still accepts the remaining discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("a failed disk enumeration must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the failed enumeration must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
tokio::fs::remove_file(temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET).join(sha_dir))
|
||||
.await
|
||||
.expect("remove the invalid directory entry");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after repair");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_data_movement_uploads() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-data-movement";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()),
|
||||
);
|
||||
set_disks
|
||||
.new_multipart_upload(
|
||||
bucket,
|
||||
"object.bin",
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create data movement upload");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_unreadable_ownership() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-corrupt";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload");
|
||||
let mut corrupted = 0;
|
||||
for temp_dir in &temp_dirs {
|
||||
for path in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
tokio::fs::write(path, b"invalid upload metadata")
|
||||
.await
|
||||
.expect("corrupt upload metadata");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
assert!(corrupted > 0, "upload metadata must exist before corruption");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("unreadable upload ownership must block cleanup");
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): a single upload directory whose `xl.meta` was
|
||||
/// destroyed (crash mid-write, torn disk state) must degrade to that upload
|
||||
/// alone. Failing the whole ListMultipartUploads turns one piece of stale
|
||||
|
||||
@@ -4184,19 +4184,7 @@ impl SetDisks {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None, reservations)
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, opts, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -4215,7 +4203,7 @@ impl SetDisks {
|
||||
// The owned publication capability supplies the exact target write
|
||||
// domain at commit time; the generic PUT path must not reacquire it.
|
||||
opts.no_lock = true;
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence), None)
|
||||
self.put_object_with_old_current_size_boxed(bucket, object, data, &opts, Some(publication_fence))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -4226,9 +4214,8 @@ impl SetDisks {
|
||||
data: &'a mut PutObjReader,
|
||||
opts: &'a ObjectOptions,
|
||||
publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> impl Future<Output = Result<(ObjectInfo, Option<OldCurrentSize>)>> + Send + 'a {
|
||||
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence, reservations))
|
||||
Box::pin(self.put_object_with_old_current_size_inner(bucket, object, data, opts, publication_fence))
|
||||
}
|
||||
|
||||
async fn put_object_with_old_current_size_inner(
|
||||
@@ -4238,7 +4225,6 @@ impl SetDisks {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
mut publication_fence: Option<RemoteTuplePublicationFence>,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let write_context = WriteCommitContext::from_options(opts, bucket, object, publication_fence.is_some())?;
|
||||
let protect_write = opts.shard_integrity_write_enabled();
|
||||
@@ -4548,7 +4534,7 @@ impl SetDisks {
|
||||
|
||||
let stream = mem::replace(
|
||||
&mut data.stream,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, false)?,
|
||||
HashReader::from_stream(Cursor::new(Vec::new()), 0, 0, None, None, true)?,
|
||||
);
|
||||
|
||||
let small_size_hint = if matches!(write_path, SmallWritePath::Inline | SmallWritePath::SingleBlockNonInline) {
|
||||
@@ -4803,83 +4789,6 @@ impl SetDisks {
|
||||
.await?;
|
||||
}
|
||||
|
||||
// A new version has no outer bucket snapshot. Prepare its remote
|
||||
// owners after staging, before taking lifecycle or object locks.
|
||||
let prepare_new_version = reservations.is_none()
|
||||
&& !opts.overwrites_existing_version()
|
||||
&& !self.ctx.lock_manager().is_disabled()
|
||||
&& crate::set_disk::put_rename_early_ack_enabled()
|
||||
&& write_context.acquires_namespace()
|
||||
&& write_context.allows_early_ack(opts.write_completion)
|
||||
&& publication_fence.is_none()
|
||||
&& opts.scanner_publication_commit_scope.is_none()
|
||||
&& opts.decommission_capacity_admission.is_none()
|
||||
&& opts.decommission_capacity.is_none()
|
||||
&& opts.quota_admission.is_none()
|
||||
&& opts.object_lock_config_snapshot.is_none()
|
||||
&& opts.bucket_lifecycle_lock_fence.is_none()
|
||||
&& opts.namespace_lock_fence.is_none()
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& opts
|
||||
.put_object_cancellation
|
||||
.as_ref()
|
||||
.is_none_or(|token| !token.is_cancelled())
|
||||
&& rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
!= rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
|
||||
.await
|
||||
.is_some();
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if prepare_new_version {
|
||||
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
|
||||
}
|
||||
|
||||
// Destructive writes prepare before the bucket snapshot. Local-only
|
||||
// fixtures without a metadata store can prepare before the object lock.
|
||||
let prepared_metadata_reservations = if !self.ctx.lock_manager().is_disabled()
|
||||
&& crate::set_disk::put_rename_early_ack_enabled()
|
||||
&& write_context.acquires_namespace()
|
||||
&& write_context.allows_early_ack(opts.write_completion)
|
||||
&& publication_fence.is_none()
|
||||
&& opts.scanner_publication_commit_scope.is_none()
|
||||
&& opts.decommission_capacity_admission.is_none()
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
{
|
||||
if let Some(prepared) = reservations {
|
||||
crate::set_disk::align_rename_metadata_reservations(prepared, &shuffle_disks)
|
||||
} else if prepare_new_version && let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil()) {
|
||||
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
|
||||
&shuffle_disks,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
&self.ctx,
|
||||
true,
|
||||
)
|
||||
.await?
|
||||
} else if shuffle_disks.iter().flatten().all(|disk| disk.is_local())
|
||||
&& crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx)
|
||||
.await
|
||||
.is_none()
|
||||
&& let Some(incarnation) = source_bucket_incarnation_id.filter(|id| !id.is_nil())
|
||||
{
|
||||
crate::set_disk::core::io_primitives::prepare_rename_metadata_reservations(
|
||||
&shuffle_disks,
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
&self.ctx,
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -4903,9 +4812,7 @@ impl SetDisks {
|
||||
|| (write_context.acquires_namespace() && object_lock_guard.is_none() && !decommission_target_lock_covered)
|
||||
{
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if !prepare_new_version {
|
||||
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
|
||||
}
|
||||
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
|
||||
if let Some(expected_incarnation_id) = source_bucket_incarnation_id
|
||||
&& opts.bucket_lifecycle_lock_fence.is_none()
|
||||
{
|
||||
@@ -5361,11 +5268,6 @@ impl SetDisks {
|
||||
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
|
||||
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
|
||||
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
|
||||
let commit_object_lock_snapshot = opts
|
||||
.object_lock_config_snapshot
|
||||
.as_ref()
|
||||
.filter(|snapshot| snapshot.owns_lifecycle_guard())
|
||||
.cloned();
|
||||
// A scanner publication scope owns the movement permit until the
|
||||
// complete rename fan-out drains. Keep this path synchronous so
|
||||
// its terminal state is known before the coordinator releases
|
||||
@@ -5379,26 +5281,6 @@ impl SetDisks {
|
||||
&& !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& commit_owns_namespace_guard
|
||||
&& commit_scanner_publication_scope.is_none();
|
||||
let lifecycle_owned = commit_bucket_lifecycle_guard.is_some()
|
||||
|| commit_object_lock_snapshot
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| snapshot.owns_lifecycle_guard_for(bucket, source_bucket_incarnation_id));
|
||||
let reservation_ctx = (commit_allows_early_ack
|
||||
&& write_context.acquires_namespace()
|
||||
&& commit_object_lock_guard.is_some()
|
||||
&& commit_decommission_object_lock_guard.is_none()
|
||||
&& commit_decommission_capacity_guard.is_none()
|
||||
&& commit_publication_guard.is_none()
|
||||
&& !quota_mutation_fence
|
||||
&& !opts.data_movement
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& lifecycle_owned)
|
||||
.then(|| Arc::clone(&self.ctx));
|
||||
let prepared_metadata_reservations = if reservation_ctx.is_some() {
|
||||
prepared_metadata_reservations
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// Full-tail callers also transfer owned guards to the coordinator:
|
||||
// cancelling their ACK waiter must not cancel an in-flight rename.
|
||||
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|
||||
@@ -5722,7 +5604,6 @@ impl SetDisks {
|
||||
)
|
||||
.with_publication_scope(commit_scanner_publication_scope.clone())
|
||||
.with_rollback_receipt(commit_rollback_receipt.clone())
|
||||
.with_reservation_context(reservation_ctx, prepared_metadata_reservations)
|
||||
.with_namespace_commit_guard(
|
||||
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
),
|
||||
@@ -5746,7 +5627,6 @@ impl SetDisks {
|
||||
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameQuorum).await;
|
||||
}
|
||||
let mut rename_guard_release = None;
|
||||
let mut committed_tail_guard = None;
|
||||
let mut needs_immediate_heal = false;
|
||||
let mut tail_owns_tmp_cleanup = false;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -5778,14 +5658,7 @@ impl SetDisks {
|
||||
.or_else(|| commit_version_suspended.then(Uuid::nil))
|
||||
.map(|version_id| version_id.to_string());
|
||||
request.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
let object_lock_guard = if rename_commit.tail_has_metadata_reservations {
|
||||
None
|
||||
} else {
|
||||
_object_lock_guard
|
||||
.take()
|
||||
.map(|guard| Arc::new(tokio::sync::Mutex::new(guard)))
|
||||
};
|
||||
committed_tail_guard = object_lock_guard.clone();
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let publication_guard = _publication_guard.take();
|
||||
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
|
||||
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
|
||||
@@ -5810,7 +5683,6 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
commit_object_lock_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5832,7 +5704,6 @@ impl SetDisks {
|
||||
write_context,
|
||||
publication_guard,
|
||||
bucket_lifecycle_guard,
|
||||
object_lock_snapshot,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
@@ -5846,7 +5717,6 @@ impl SetDisks {
|
||||
drop(write_context);
|
||||
drop(publication_guard);
|
||||
drop(bucket_lifecycle_guard);
|
||||
drop(object_lock_snapshot);
|
||||
cleanup_runtime::run(cleanup_reservation, async move {
|
||||
let zero_target_cleanup = targets.is_empty();
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
@@ -6102,13 +5972,6 @@ impl SetDisks {
|
||||
lock_held_post_rename_started,
|
||||
);
|
||||
let lock_held_guard_release_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Some(guard) = committed_tail_guard.take() {
|
||||
// The committed tail still owns every physical rename. Convert
|
||||
// only after metadata invalidation and before acknowledging PUT.
|
||||
// Unsupported peers keep their exclusive leases and the tail
|
||||
// retains ownership until its physical work has drained.
|
||||
let _ = guard.lock().await.downgrade_committed_tail().await;
|
||||
}
|
||||
if let Some(release) = rename_guard_release.take() {
|
||||
let _ = release.send(true);
|
||||
}
|
||||
@@ -22318,117 +22181,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_retains_owned_bucket_lifecycle_fence() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-tail-lifecycle";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
let lock = NamespaceLock::new("tail-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
Uuid::new_v4(),
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let opts = ObjectOptions {
|
||||
object_lock_config_snapshot: Some(snapshot),
|
||||
..Default::default()
|
||||
};
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer_set = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
writer_set
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename tail pauses");
|
||||
put.await.expect("PUT joins").expect("PUT acknowledged");
|
||||
let mut bucket_delete =
|
||||
Box::pin(lock.get_write_lock(ObjectKey::new(bucket, "lifecycle"), "delete", Duration::from_secs(10)));
|
||||
let early = tokio::time::timeout(Duration::from_secs(2), &mut bucket_delete).await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let guard = match early {
|
||||
Ok(result) => {
|
||||
drop(result.expect("early bucket lock result"));
|
||||
panic!("bucket lifecycle owner must outlive the acknowledged physical tail");
|
||||
}
|
||||
Err(_) => tokio::time::timeout(Duration::from_secs(15), bucket_delete)
|
||||
.await
|
||||
.expect("lifecycle lock released after tail")
|
||||
.expect("bucket delete acquires lock"),
|
||||
};
|
||||
drop(guard);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_drain_allows_read_after_acknowledged_put() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
|
||||
],
|
||||
async {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-early-ack-tail-read";
|
||||
let object = "early-ack-tail-read-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let put_set = Arc::clone(&set_disks);
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
put_set
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("tail disk should pause during rename");
|
||||
put.await
|
||||
.expect("PUT task should join")
|
||||
.expect("PUT should be acknowledged at quorum");
|
||||
let result = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await;
|
||||
let mut reader = result.expect("acknowledged object should remain readable while tail drains");
|
||||
let mut body = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(5), reader.stream.read_to_end(&mut body))
|
||||
.await
|
||||
.expect("object body should not wait for the tail")
|
||||
.expect("object body should drain");
|
||||
assert_eq!(body, vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
rename_barrier.release();
|
||||
drop(rename_barrier);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_drain_retains_namespace_lock_until_background_rename_finishes() {
|
||||
@@ -22501,200 +22253,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_does_not_timeout_the_next_put() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-early-ack-next-writer";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
{
|
||||
let map = set.ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
let lifecycle_lock = NamespaceLock::new("next-put-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lifecycle_lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lifecycle_lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let incarnation = Uuid::new_v4();
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
incarnation,
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let mut opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
object_lock_config_snapshot: Some(snapshot.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_set = Arc::clone(&set);
|
||||
let first_opts = opts.clone();
|
||||
let first = tokio::spawn(async move {
|
||||
first_set
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &first_opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first rename paused");
|
||||
first.await.expect("first PUT joins").expect("first PUT acknowledged");
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]), &opts),
|
||||
)
|
||||
.await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let second = second.expect("second PUT finishes").expect("second PUT must not time out");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("both rename tails must drain");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read newest object");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("read full body");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
let expected_etag = second.etag.as_ref().expect("second PUT ETag");
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("each disk must retain the latest version");
|
||||
assert_eq!(info.metadata.get("etag"), Some(expected_etag), "late rename must not restore an old copy");
|
||||
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
|
||||
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_tail_cannot_restore_a_deleted_version() {
|
||||
use rustfs_lock::{LocalClient, NamespaceLock, ObjectKey};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-tail-version-delete";
|
||||
let object = "object";
|
||||
for disk in &disks {
|
||||
disk.make_volume(bucket).await.expect("create volume");
|
||||
}
|
||||
{
|
||||
let map = set.ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
let lock = NamespaceLock::new("version-delete-lifecycle".to_string(), Arc::new(LocalClient::new()));
|
||||
let lifecycle = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "lifecycle"), "lifecycle", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("lifecycle read lock");
|
||||
let metadata = lock
|
||||
.get_read_lock(ObjectKey::new(bucket, "metadata"), "metadata", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("metadata read lock");
|
||||
let incarnation = Uuid::new_v4();
|
||||
let snapshot = Arc::new(crate::object_api::ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
Uuid::new_v4(),
|
||||
bucket,
|
||||
incarnation,
|
||||
OffsetDateTime::now_utc(),
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
lifecycle,
|
||||
metadata,
|
||||
));
|
||||
let mut opts = ObjectOptions {
|
||||
versioned: true,
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
object_lock_config_snapshot: Some(snapshot.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.add_lock_fences(&mut opts);
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer = Arc::clone(&set);
|
||||
let put_opts = opts.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
writer
|
||||
.put_object(bucket, object, &mut PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]), &put_opts)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("rename pauses");
|
||||
let version = put
|
||||
.await
|
||||
.expect("PUT joins")
|
||||
.expect("PUT succeeds")
|
||||
.version_id
|
||||
.expect("version ID")
|
||||
.to_string();
|
||||
let delete_barrier = DeleteObjectCommitBarrier::install_before_namespace(bucket, object);
|
||||
let delete_set = Arc::clone(&set);
|
||||
let mut delete_opts = opts;
|
||||
delete_opts.version_id = Some(version.clone());
|
||||
let delete = tokio::spawn(async move { delete_set.delete_object(bucket, object, delete_opts).await });
|
||||
delete_barrier.wait_until_paused().await;
|
||||
delete_barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !delete_barrier.namespace_acquired() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("version delete acquires namespace while old tail is paused");
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
delete.await.expect("delete joins").expect("version delete succeeds");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("rename tails drain");
|
||||
for disk in &disks {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.read_version("", bucket, object, &version, &ReadOptions::default()).await,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)
|
||||
),
|
||||
"late tail must not restore the deleted version"
|
||||
);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn early_ack_successful_tail_reclaims_its_old_non_inline_data_dir() {
|
||||
|
||||
@@ -44,15 +44,16 @@ pub(crate) mod lifecycle {
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker, WalkOptions as StorageWalkOptions,
|
||||
WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker,
|
||||
WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
|
||||
MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -523,7 +523,22 @@ impl ECStore {
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, None, validate).await
|
||||
}
|
||||
|
||||
/// Validate an external fence and the bucket incarnation in one metadata transaction.
|
||||
pub async fn update_bucket_metadata_config_for_incarnation_validated<F>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, Some(expected_incarnation_id), validate).await
|
||||
}
|
||||
|
||||
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
@@ -2279,17 +2294,8 @@ mod tests {
|
||||
drop(
|
||||
lock.get_write_lock(Duration::from_secs(30))
|
||||
.await
|
||||
.expect("fixture object lock should be available"),
|
||||
.expect("fixture rename tail should finish before checking its generation"),
|
||||
);
|
||||
// A quorum-acknowledged rename can release the object lock while its
|
||||
// physical tail still holds the instance-wide namespace owner.
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while ecstore.ctx.namespace_commits_pending() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("fixture namespace owners should drain before checking its generation");
|
||||
assert_eq!(
|
||||
ecstore.scanner_namespace_mutation_generation(),
|
||||
generation_before_put.saturating_add(3),
|
||||
|
||||
@@ -45,8 +45,7 @@
|
||||
use crate::error::{Result, StorageError};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
use std::time::Duration;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use rustfs_lock::NamespaceLockGuard;
|
||||
use rustfs_lock::distributed_lock::LockLostSignal;
|
||||
@@ -99,74 +98,9 @@ pub(super) struct FenceRegistration {
|
||||
#[derive(Default)]
|
||||
pub(crate) struct BucketFenceRegistry {
|
||||
entries: Mutex<HashMap<String, FenceEntry>>,
|
||||
reservation_slots: Mutex<HashMap<(String, Uuid), Weak<ReservationFenceSlot>>>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ReservationFenceSlot {
|
||||
pub(super) owner: tokio::sync::Mutex<Weak<ReservationFenceGuard>>,
|
||||
}
|
||||
|
||||
/// Shared by short bursts of reservations, but never kept alive by the cache.
|
||||
pub(super) struct ReservationFenceGuard {
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
created_at: tokio::time::Instant,
|
||||
_slot: Arc<ReservationFenceSlot>,
|
||||
}
|
||||
|
||||
impl ReservationFenceGuard {
|
||||
pub(super) fn new(
|
||||
publication: NamespaceLockGuard,
|
||||
lifecycle: BucketIncarnationFenceGuard,
|
||||
slot: Arc<ReservationFenceSlot>,
|
||||
) -> Self {
|
||||
Self {
|
||||
publication,
|
||||
lifecycle,
|
||||
created_at: tokio::time::Instant::now(),
|
||||
_slot: slot,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn is_reusable(&self) -> bool {
|
||||
// Stop admitting new reservations through this shared reader so a
|
||||
// queued bucket writer can progress after the existing tails drain.
|
||||
self.created_at.elapsed() < Duration::from_secs(1) && self.is_held()
|
||||
}
|
||||
|
||||
fn is_held(&self) -> bool {
|
||||
!self.publication.is_lock_lost()
|
||||
&& !self.publication.is_released()
|
||||
&& !self.lifecycle.is_lock_lost()
|
||||
&& self
|
||||
.lifecycle
|
||||
.namespace_lock_guard()
|
||||
.is_some_and(|guard| !guard.is_released())
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::disk::os::MetadataReservationFence for ReservationFenceGuard {
|
||||
fn is_valid(&self) -> bool {
|
||||
self.is_held()
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketFenceRegistry {
|
||||
pub(super) fn reservation_slot(&self, bucket: &str, incarnation: Uuid) -> Arc<ReservationFenceSlot> {
|
||||
let mut slots = self.reservation_slots.lock().expect("reservation fence registry poisoned");
|
||||
let key = (bucket.to_owned(), incarnation);
|
||||
if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
|
||||
return slot;
|
||||
}
|
||||
if slots.len() >= 1024 {
|
||||
slots.retain(|_, slot| slot.strong_count() != 0);
|
||||
}
|
||||
let slot = Arc::new(ReservationFenceSlot::default());
|
||||
slots.insert(key, Arc::downgrade(&slot));
|
||||
slot
|
||||
}
|
||||
|
||||
/// Register a new live guard for `bucket` and return the memoized
|
||||
/// incarnation id if one is valid for the current coverage window.
|
||||
fn enter(&self, bucket: &str, loss_probe: LockLossProbe) -> FenceRegistration {
|
||||
|
||||
@@ -3692,6 +3692,177 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
// The RPC lock service uses the raw ObjectKey and one backend per node,
|
||||
// even when the requesting sets carry different pool namespace strings.
|
||||
async fn pool_meta_heal_store_with_lock_domains(source: &Arc<ECStore>, shared: bool) -> Arc<ECStore> {
|
||||
use rustfs_lock::client::{LockClient, local::LocalClient};
|
||||
|
||||
let clients = || {
|
||||
(0..2)
|
||||
.map(|_| {
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new()))) as Arc<dyn LockClient>
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let first = clients();
|
||||
let second = if shared { first.clone() } else { clients() };
|
||||
let mut pools = source.pools.clone();
|
||||
for (pool, lockers) in pools.iter_mut().zip([first, second]) {
|
||||
for set in &mut Arc::make_mut(pool).disk_set {
|
||||
Arc::make_mut(set).lockers = lockers.clone();
|
||||
}
|
||||
}
|
||||
let endpoints: EndpointServerPools = pools.iter().map(|pool| pool.endpoints.clone()).collect::<Vec<_>>().into();
|
||||
let state = source.pool_meta_save_gate.lock().await.independent_clone_for_test();
|
||||
Arc::new(ECStore {
|
||||
id: Uuid::new_v4(),
|
||||
disk_map: Default::default(),
|
||||
pools,
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoints, Arc::clone(&source.ctx)),
|
||||
pool_meta: tokio::sync::RwLock::new(source.pool_meta.read().await.clone()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(None),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; 2]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(state),
|
||||
ctx: Arc::clone(&source.ctx),
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_same_distributed_backend_has_one_raw_lock_across_pools() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let first = store.pools[0].get_disks_by_key(POOL_META_NAME);
|
||||
let second = store.pools[1].get_disks_by_key(POOL_META_NAME);
|
||||
let first_lock = first
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("first pool lock");
|
||||
let second_lock = second
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("second pool lock");
|
||||
let guard = first_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(
|
||||
second_lock
|
||||
.get_write_lock(std::time::Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"different pool namespaces still conflict on the same backend and object key"
|
||||
);
|
||||
drop(guard);
|
||||
second_lock
|
||||
.get_write_lock(std::time::Duration::from_secs(3))
|
||||
.await
|
||||
.expect("second writer should acquire after the first releases");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_reuses_same_distributed_backend_across_pools() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let (item, error) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
store.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("inspection must reuse the fixed pool's writer when the lock backend is shared")
|
||||
.expect("pool metadata heal inspection should return a result");
|
||||
assert!(error.is_none(), "inspection should succeed: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_repairs_other_pool_with_shared_distributed_backend() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, true).await;
|
||||
let missing = remove_pool_meta_shard(&store, 1).await;
|
||||
let (item, error) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
store.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("cross-pool repair must not reenter the shared writer")
|
||||
.expect("pool metadata repair should return a result");
|
||||
assert!(error.is_none(), "cross-pool repair should complete: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
assert!(
|
||||
missing.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
|
||||
"the selected missing shard must be restored"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_keeps_target_lock_for_independent_distributed_backend() {
|
||||
let (_dirs, source, _peer) = test_two_pool_stores(None).await;
|
||||
let store = pool_meta_heal_store_with_lock_domains(&source, false).await;
|
||||
let target = store.pools[1].get_disks_by_key(POOL_META_NAME);
|
||||
let lock = target
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
|
||||
.await
|
||||
.expect("target lock");
|
||||
let guard = lock
|
||||
.get_write_lock(std::time::Duration::from_secs(2))
|
||||
.await
|
||||
.expect("hold target lock");
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
let heal_store = Arc::clone(&store);
|
||||
let mut heal = tokio::spawn(async move {
|
||||
heal_store
|
||||
.handle_heal_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
POOL_META_NAME,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(3),
|
||||
barrier.wait_until_external_heal_target_lock_attempted(),
|
||||
)
|
||||
.await
|
||||
.expect("inspection must attempt the independent target lock");
|
||||
assert!(!heal.is_finished(), "inspection must wait while the target writer is held");
|
||||
drop(guard);
|
||||
let (item, error) = tokio::time::timeout(std::time::Duration::from_secs(3), &mut heal)
|
||||
.await
|
||||
.expect("inspection should progress after target lock release")
|
||||
.expect("heal task should not panic")
|
||||
.expect("pool metadata heal inspection should return a result");
|
||||
assert!(error.is_none(), "inspection should succeed: {error:?}");
|
||||
assert_eq!(item.object, POOL_META_NAME);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_heal_cas_repairs_only_the_requested_pool() {
|
||||
|
||||
@@ -37,8 +37,8 @@ use crate::error::{
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::list::{
|
||||
ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr,
|
||||
StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo,
|
||||
StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
@@ -80,6 +80,21 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl AuthoritativeListOperations for ECStore {
|
||||
async fn walk_authoritative(
|
||||
self: Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
check_list_objs_args(bucket, prefix, &None)?;
|
||||
self.walk_authoritatively(rx, bucket, prefix, result, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the versioned listing options used by recursive object walks.
|
||||
///
|
||||
/// `list_path_result` normally derives these scan bounds before dispatching to
|
||||
@@ -1550,6 +1565,57 @@ impl ECStore {
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn walk_authoritatively(
|
||||
self: Arc<Self>,
|
||||
cancellation: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
let cancel = cancellation.child_token();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let mut scans = Vec::new();
|
||||
let mut inputs = Vec::new();
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
inputs.push(receiver);
|
||||
let mut scan = recursive_walk_list_path_options(bucket.to_owned(), prefix, &opts);
|
||||
// Internal absence checks must include reserved object prefixes.
|
||||
scan.skip_hidden_prefix_check = true;
|
||||
scans.push(set.list_versions_authoritatively(cancel.clone(), scan, sender));
|
||||
}
|
||||
}
|
||||
if scans.is_empty() {
|
||||
return Err(StorageError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let (sender, receiver) = mpsc::channel(100);
|
||||
let forwarder = spawn_walk_entry_forwarder(cancel.clone(), receiver, result, opts, bucket.to_owned());
|
||||
let merge = spawn_listing_merge(cancel.clone(), inputs, sender);
|
||||
let scan_results = join_all(scans).await;
|
||||
let merge_result = merge.await.map_err(Error::from)?;
|
||||
let forward_result = forwarder.await.map_err(Error::from);
|
||||
|
||||
// An unavailable set cannot establish absence, even when another set
|
||||
// completed. Only an explicitly missing directory or volume is empty.
|
||||
for scan in scan_results {
|
||||
match scan {
|
||||
Ok(outcome) if !outcome.limit_reached => {}
|
||||
Ok(_) => return Err(StorageError::ErasureReadQuorum),
|
||||
Err(StorageError::FileNotFound | StorageError::VolumeNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
merge_result?;
|
||||
forward_result?;
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(Error::other("authoritative object walk was cancelled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Forwards merged raw entries to a walk consumer as `ObjectInfoOrErr`,
|
||||
@@ -2520,7 +2586,8 @@ impl SetDisks {
|
||||
/// Version scan for set-scoped callers that act on absence: set walks
|
||||
/// (tier reference proofs, tier-delete journals, pool maintenance) and the
|
||||
/// recursive prefix-delete scan. Every online drive is read and each
|
||||
/// version is resolved at the read quorum of its own erasure layout; a
|
||||
/// version uses the read quorum of its own erasure layout. Null versions
|
||||
/// in the system bucket also retain GET's commit-proof selection. A
|
||||
/// version that cannot be ruled out fails the scan instead of being
|
||||
/// omitted. Merged `ListObjectVersions` pages go through `list_path`.
|
||||
async fn list_versions_authoritatively(
|
||||
@@ -2859,6 +2926,7 @@ mod test {
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::{SetDisks, hermetic_set_disks_isolated};
|
||||
use crate::storage_api_contracts::list::AuthoritativeListOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use bytes::Bytes;
|
||||
@@ -5719,6 +5787,126 @@ mod test {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_rejects_uncommitted_null_version_split() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
for new_etag in ["new", "old"] {
|
||||
let new =
|
||||
test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), new_etag, Uuid::from_u128(2));
|
||||
let entries = MetaCacheEntries(vec![Some(old.clone()), Some(old.clone()), Some(new.clone()), Some(new)]);
|
||||
|
||||
assert!(matches!(
|
||||
SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2),
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_selects_committed_null_version() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (new_copies, expected) in [(1, "old"), (3, "new"), (4, "new")] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| Some(if slot < new_copies { new.clone() } else { old.clone() }))
|
||||
.collect(),
|
||||
);
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2)
|
||||
.expect("committed generation should resolve")
|
||||
.expect("committed record should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_matches_degraded_null_version_reads() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (old_copies, new_copies, expected) in [
|
||||
(2, 1, Some("old")),
|
||||
(1, 2, None),
|
||||
(0, 3, Some("new")),
|
||||
(0, 2, Some("new")),
|
||||
(0, 1, None),
|
||||
] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| {
|
||||
if slot < old_copies {
|
||||
Some(old.clone())
|
||||
} else if slot < old_copies + new_copies {
|
||||
Some(new.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let errors = (0..4)
|
||||
.map(|slot| (slot >= old_copies + new_copies).then_some(DiskError::DiskNotFound))
|
||||
.collect::<Vec<_>>();
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2);
|
||||
|
||||
if let Some(expected) = expected {
|
||||
let info = resolved
|
||||
.expect("degraded committed record should resolve")
|
||||
.expect("degraded committed record should be present")
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
} else {
|
||||
assert!(matches!(resolved, Err(DiskError::ErasureReadQuorum)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_user_bucket_scan_keeps_null_version_read_quorum_selection() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for old_copies in [1, 2] {
|
||||
let entries = MetaCacheEntries(vec![
|
||||
Some(new.clone()),
|
||||
Some(new.clone()),
|
||||
Some(old.clone()),
|
||||
(old_copies == 2).then(|| old.clone()),
|
||||
]);
|
||||
let errors = [None, None, None, (old_copies == 1).then_some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions("bucket", entries, &errors, 4, 2)
|
||||
.expect("user bucket read quorum should resolve")
|
||||
.expect("user bucket record should be present");
|
||||
let info = resolved.to_fileinfo("bucket").expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_keeps_explicit_version_read_quorum() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let entry = test_object_meta_entry_with_erasure_versions("entry.json", &[(time, "version", 2, 2)]);
|
||||
let entries = MetaCacheEntries(vec![Some(entry.clone()), Some(entry), None, None]);
|
||||
let errors = [None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2)
|
||||
.expect("explicit version read quorum should resolve")
|
||||
.expect("explicit version should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_version_scan_tolerates_corrupt_replicas_only_with_version_quorum() {
|
||||
let entry = test_object_meta_entry_with_erasure_versions(
|
||||
@@ -6060,6 +6248,49 @@ mod test {
|
||||
Bytes::from(meta.marshal_msg().expect("encode fixture xl.meta"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_store_walk_includes_reserved_prefix_versions() {
|
||||
let bucket = "catalog-cleanup-versions";
|
||||
let name = ".rustfs-table/namespaces/ns/tables/table/metadata/metadata.json";
|
||||
let (_dirs, store) = single_set_store(16).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
for (slot, disk) in set.disks.read().await.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("fixture drive is online");
|
||||
disk.make_volume(bucket).await.expect("create bucket volume");
|
||||
disk.write_all(
|
||||
bucket,
|
||||
&format!("{name}/{STORAGE_FORMAT_FILE}"),
|
||||
versioned_object_metadata(bucket, name, &[(2, 1_700_000_002), (1, 1_700_000_001)], slot),
|
||||
)
|
||||
.await
|
||||
.expect("persist reserved metadata versions");
|
||||
}
|
||||
let (sender, mut receiver) = mpsc::channel(1);
|
||||
let walk = store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let collect = async {
|
||||
let mut versions = Vec::new();
|
||||
while let Some(item) = receiver.recv().await {
|
||||
assert!(item.err.is_none(), "reserved metadata must decode: {:?}", item.err);
|
||||
let item = item.item.expect("walk must return an object");
|
||||
versions.push((item.name, item.version_id));
|
||||
}
|
||||
versions
|
||||
};
|
||||
let (walk, versions) = timeout(Duration::from_secs(10), async { tokio::join!(walk, collect) })
|
||||
.await
|
||||
.expect("authoritative reserved-prefix walk must terminate");
|
||||
walk.expect("authoritative walk must succeed");
|
||||
assert_eq!(
|
||||
versions,
|
||||
[
|
||||
(name.to_string(), Some(Uuid::from_u128(1))),
|
||||
(name.to_string(), Some(Uuid::from_u128(2)))
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// A versioned DeleteObject(versionId) can reach write quorum while three
|
||||
// drives are offline; they keep the deleted version after they return. A
|
||||
// different drive going offline later must not turn that sub-quorum
|
||||
@@ -6130,6 +6361,16 @@ mod test {
|
||||
.await
|
||||
.expect("set walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let (sender, mut receiver) = mpsc::channel(16);
|
||||
let walk =
|
||||
store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let drain = async { while receiver.recv().await.is_some() {} };
|
||||
let (walk, ()) = timeout(Duration::from_secs(10), async { tokio::join!(walk, drain) })
|
||||
.await
|
||||
.expect("authoritative store walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let recursive_delete_scan = set
|
||||
.clone()
|
||||
.inner_list_object_versions_for_recursive_delete(bucket, "", None, None, 1000)
|
||||
|
||||
@@ -456,7 +456,6 @@ mod rebalance;
|
||||
pub(crate) use rebalance::PoolSpaceForTest;
|
||||
pub(crate) mod utils;
|
||||
|
||||
pub(crate) use peer::admit_local_disk;
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
|
||||
|
||||
@@ -19,10 +19,28 @@ use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
use futures::{StreamExt, stream};
|
||||
use rustfs_utils::path::encode_dir_object_ref;
|
||||
use std::collections::HashSet;
|
||||
|
||||
const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::multipart::MultipartUploadInspection for ECStore {
|
||||
type Error = Error;
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
let (_, lifecycle_guard) = self
|
||||
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
|
||||
.await?;
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
set.ensure_no_multipart_uploads(bucket).await?;
|
||||
}
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_guard_held(lifecycle_guard.as_ref(), bucket, "")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
|
||||
std::sync::OnceLock::new();
|
||||
@@ -1168,11 +1186,9 @@ impl ECStore {
|
||||
let oi = if require_all_pools {
|
||||
lookup_opts.metadata_cache_safe = false;
|
||||
lookup_opts.suppress_read_repair = true;
|
||||
if let Err(err) = check_object_args(bucket, object) {
|
||||
return Some(err);
|
||||
}
|
||||
let encoded_object = encode_dir_object(object);
|
||||
self.get_latest_object_info_for_conditional_write(bucket, &encoded_object, &lookup_opts)
|
||||
// Only completion uses this branch, after validating bucket and object.
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
self.get_latest_object_info_for_conditional_write(bucket, encoded_object.as_ref(), &lookup_opts)
|
||||
.await
|
||||
.and_then(|(info, _)| {
|
||||
lookup_opts.precondition_check(&info)?;
|
||||
|
||||
@@ -4814,45 +4814,7 @@ impl ECStore {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let mut early_opts = opts.clone();
|
||||
// Bucket publication precedes lifecycle and metadata transaction locks.
|
||||
let prepared = if self.single_pool()
|
||||
&& !self.ctx.lock_manager().is_disabled()
|
||||
&& crate::set_disk::put_rename_early_ack_enabled()
|
||||
&& early_opts.write_completion == crate::object_api::WriteCompletion::Quorum
|
||||
&& early_opts.overwrites_existing_version()
|
||||
&& !is_meta_bucketname(bucket)
|
||||
&& !early_opts.no_lock
|
||||
&& !early_opts.data_movement
|
||||
&& early_opts.write_lock_context.is_none()
|
||||
&& early_opts.object_lock_config_snapshot.is_none()
|
||||
&& early_opts.bucket_lifecycle_lock_fence.is_none()
|
||||
&& early_opts.namespace_lock_fence.is_none()
|
||||
&& early_opts.scanner_publication_commit_scope.is_none()
|
||||
&& early_opts.decommission_capacity_admission.is_none()
|
||||
&& early_opts.decommission_capacity.is_none()
|
||||
&& early_opts.quota_admission.is_none()
|
||||
&& early_opts
|
||||
.put_object_cancellation
|
||||
.as_ref()
|
||||
.is_none_or(|token| !token.is_cancelled())
|
||||
&& rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
!= rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
{
|
||||
check_new_object_args(bucket, object, false)?;
|
||||
let incarnation = match early_opts.expected_bucket_incarnation_id {
|
||||
Some(incarnation) => incarnation,
|
||||
None => self.bucket_incarnation_id(bucket).await?,
|
||||
};
|
||||
early_opts.expected_bucket_incarnation_id = Some(incarnation);
|
||||
let encoded = encode_dir_object(object);
|
||||
let set = self.pools[0].get_disks_by_key(&encoded);
|
||||
let disks = set.disks.read().await.clone();
|
||||
crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, &encoded, incarnation, &self.ctx, true).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, &early_opts)).await?;
|
||||
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, opts)).await?;
|
||||
let idx = self
|
||||
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
|
||||
.await?;
|
||||
@@ -4861,8 +4823,7 @@ impl ECStore {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
|
||||
let result =
|
||||
Box::pin(self.pools[idx].put_object_with_prepared_reservations(bucket, object.as_str(), data, &opts, prepared)).await;
|
||||
let result = Box::pin(self.pools[idx].put_object_with_old_current_size(bucket, object.as_str(), data, &opts)).await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
match result {
|
||||
Ok((object_info, old_current_size)) => {
|
||||
@@ -9977,458 +9938,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn ecstore_early_ack_prepares_before_snapshot_and_orders_the_next_put() {
|
||||
use crate::disk::ReadOptions;
|
||||
use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "ecstore-reservation-order";
|
||||
let object = "object";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_store = Arc::clone(&store);
|
||||
let first = tokio::spawn(async move {
|
||||
first_store
|
||||
.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'1'; 1 << 20]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first tail pauses");
|
||||
first.await.expect("first task").expect("first PUT");
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
store.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'2'; 1 << 20]),
|
||||
&ObjectOptions::default(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("rename tails drain");
|
||||
let expected_etag = second.etag.as_ref().expect("second ETag");
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read latest metadata");
|
||||
assert_eq!(info.metadata.get("etag"), Some(expected_etag));
|
||||
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
|
||||
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn ecstore_new_version_early_ack_orders_the_next_put() {
|
||||
use crate::disk::ReadOptions;
|
||||
use crate::set_disk::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "ecstore-versioned-reservation-order";
|
||||
let object = "object";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_store = Arc::clone(&store);
|
||||
let first = tokio::spawn(async move {
|
||||
first_store
|
||||
.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'1'; 1 << 20]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first tail pauses");
|
||||
first.await.expect("first task").expect("first PUT");
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
store.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'2'; 1 << 20]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
let (second, _) = second.expect("second PUT completes").expect("second PUT succeeds");
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("rename tails drain");
|
||||
let expected_etag = second.etag.as_ref().expect("second ETag");
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read latest metadata");
|
||||
assert_eq!(info.metadata.get("etag"), Some(expected_etag));
|
||||
let parts = disk.check_parts(bucket, object, &info).await.expect("check latest shards");
|
||||
assert!(parts.results.iter().all(|result| *result == crate::disk::CHECK_PART_SUCCESS));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn local_prepared_reservations_hold_bucket_publication_fence() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.clone();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in disks.iter().flatten() {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "local-reservation-bucket-fence";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let reservations =
|
||||
crate::set_disk::prepare_rename_metadata_reservations(&disks, bucket, "object", incarnation, &ctx, true)
|
||||
.await
|
||||
.expect("prepare reservations")
|
||||
.expect("all disks prepared");
|
||||
let lock = store
|
||||
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("publication lock");
|
||||
let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(tokio::time::timeout(Duration::from_millis(50), &mut writer).await.is_err());
|
||||
drop(reservations);
|
||||
let guard = tokio::time::timeout(Duration::from_secs(5), writer)
|
||||
.await
|
||||
.expect("reservations release fence")
|
||||
.expect("writer lock");
|
||||
drop(guard);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn long_staged_put_does_not_timeout_the_next_overwrite() {
|
||||
use crate::set_disk::{
|
||||
ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, PutObjectCommitBarrier, PutObjectCommitPause, rename_fanout_barrier,
|
||||
};
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("true"))], async {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "long-staged-reservation";
|
||||
let object = "object";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let staging_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
|
||||
let tail_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let writer = Arc::clone(&store);
|
||||
let first = tokio::spawn(async move {
|
||||
writer
|
||||
.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'1'; 1 << 20]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
});
|
||||
staging_barrier.wait_until_paused().await;
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(25)).await;
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::advance(Duration::from_secs(26)).await;
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::resume();
|
||||
staging_barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(30), tail_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("first tail pauses");
|
||||
first.await.expect("first task").expect("first PUT");
|
||||
drop(staging_barrier);
|
||||
let second = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
store.handle_put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![b'2'; 1 << 20]),
|
||||
&ObjectOptions::default(),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
tail_barrier.release();
|
||||
drop(tail_barrier);
|
||||
assert!(matches!(second, Ok(Ok(_))), "second PUT after a long staging interval failed: {second:?}");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bucket_reservations_do_not_exhaust_shared_lock_holders() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disks = set.disks.read().await.iter().flatten().cloned().collect::<Vec<_>>();
|
||||
{
|
||||
let map = ctx.local_disk_map();
|
||||
let mut active = map.write().await;
|
||||
for disk in &disks {
|
||||
active.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
}
|
||||
}
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "reservation-reader-capacity";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let disk = disks[0].endpoint().to_string();
|
||||
let mut tokens = Vec::new();
|
||||
let mut failure = None;
|
||||
for index in 0..256 {
|
||||
match store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&disk,
|
||||
bucket,
|
||||
&format!("object-{index}"),
|
||||
incarnation,
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(token) => tokens.push(token),
|
||||
Err(error) => {
|
||||
failure = Some((index, error));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
for token in tokens {
|
||||
assert!(
|
||||
store
|
||||
.cancel_local_object_metadata_reservation(&disk, token)
|
||||
.await
|
||||
.expect("cancel token")
|
||||
);
|
||||
}
|
||||
assert!(failure.is_none(), "bucket reservations exhausted shared lock holders: {failure:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn expired_reservation_fence_does_not_bypass_a_waiting_bucket_writer() {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
|
||||
let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], Arc::clone(&ctx)).await);
|
||||
let set = store.pools[0].get_disks_by_key("object");
|
||||
let disk = set.disks.read().await.iter().flatten().next().expect("local disk").clone();
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "reservation-writer-fairness";
|
||||
store
|
||||
.handle_make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let first = store
|
||||
.reserve_local_object_metadata_at_incarnation(&disk_ref, bucket, "first", incarnation, Duration::from_secs(5))
|
||||
.await
|
||||
.expect("first reservation");
|
||||
let lock = store
|
||||
.new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.expect("publication lock");
|
||||
let mut writer = Box::pin(lock.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut writer).await.is_err(),
|
||||
"writer waits for first token"
|
||||
);
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
tokio::time::resume();
|
||||
let mut second = Box::pin(store.reserve_local_object_metadata_at_incarnation(
|
||||
&disk_ref,
|
||||
bucket,
|
||||
"second",
|
||||
incarnation,
|
||||
Duration::from_secs(5),
|
||||
));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
|
||||
"expired cached reader must not bypass writer"
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, first)
|
||||
.await
|
||||
.expect("cancel first token")
|
||||
);
|
||||
let guard = tokio::time::timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("writer proceeds")
|
||||
.expect("writer lock");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(20), &mut second).await.is_err(),
|
||||
"new reader waits while writer holds the lock"
|
||||
);
|
||||
drop(guard);
|
||||
let token = tokio::time::timeout(Duration::from_secs(2), second)
|
||||
.await
|
||||
.expect("reader proceeds")
|
||||
.expect("second reservation");
|
||||
assert!(
|
||||
store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, token)
|
||||
.await
|
||||
.expect("cancel second token")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lost_bucket_fence_cannot_activate_a_prepared_reservation() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)));
|
||||
let client = Arc::new(RefreshLossLockClient::with_manager(manager));
|
||||
let lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
|
||||
"reservation-loss".to_string(),
|
||||
vec![client.clone() as Arc<dyn rustfs_lock::LockClient>],
|
||||
1,
|
||||
);
|
||||
let acquire = |object: &'static str| {
|
||||
rustfs_lock::LockRequest::new(
|
||||
rustfs_lock::ObjectKey::new("bucket", object),
|
||||
rustfs_lock::LockType::Shared,
|
||||
"reservation-loss",
|
||||
)
|
||||
.with_ttl(Duration::from_secs(2))
|
||||
.with_refresh_interval(Duration::from_millis(20))
|
||||
};
|
||||
let publication = lock
|
||||
.acquire_guard(&acquire("publication"))
|
||||
.await
|
||||
.expect("publication request")
|
||||
.expect("publication quorum");
|
||||
let signal = publication.lock_lost_signal().expect("distributed loss signal");
|
||||
let lifecycle = lock
|
||||
.acquire_guard(&acquire("lifecycle"))
|
||||
.await
|
||||
.expect("lifecycle request")
|
||||
.expect("lifecycle quorum");
|
||||
let bucket_registry = Arc::new(crate::store::bucket_fence::BucketFenceRegistry::default());
|
||||
let lifecycle =
|
||||
crate::store::bucket_fence::BucketIncarnationFenceGuard::new(bucket_registry.clone(), "bucket", lifecycle, true);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let slot = bucket_registry.reservation_slot("bucket", incarnation);
|
||||
let owner = Arc::new(crate::store::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot));
|
||||
let registry = crate::disk::os::MetadataReservationRegistry::new(1);
|
||||
let key = std::path::Path::new("bucket/object");
|
||||
let disk = Uuid::new_v4();
|
||||
let token = registry
|
||||
.prepare_with_owner(key, disk, Some(owner))
|
||||
.expect("prepare reservation");
|
||||
client.reject_refreshes();
|
||||
tokio::time::timeout(Duration::from_secs(5), signal.notified())
|
||||
.await
|
||||
.expect("publication lock loss");
|
||||
assert!(
|
||||
registry.activate(token, key, disk).await.is_err(),
|
||||
"a lost bucket fence must not enter the mutation queue"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn borrowed_copy_tail_drained_waits_for_rename_tail() {
|
||||
|
||||
@@ -85,190 +85,6 @@ fn validate_bootstrap_volume(volume: &str) -> DiskResult<()> {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Prepare an inert user-object reservation on the currently admitted disk.
|
||||
/// An unclaimed token expires and does not hold the object's metadata lock.
|
||||
#[cfg(test)]
|
||||
pub async fn reserve_local_object_metadata(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Prepare before the caller acquires bucket lifecycle or object locks.
|
||||
/// The token holds the bucket publication fence through physical completion.
|
||||
pub async fn reserve_local_object_metadata_at_incarnation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> DiskResult<Uuid> {
|
||||
if self.ctx.lock_manager().is_disabled()
|
||||
|| crate::bucket::utils::check_valid_bucket_name(bucket).is_err()
|
||||
|| is_meta_bucketname(bucket)
|
||||
|| incarnation.is_nil()
|
||||
|| timeout.is_zero()
|
||||
|| rustfs_utils::path::object_name_disk_identity(object).as_ref()
|
||||
== rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
tokio::time::timeout(timeout.min(std::time::Duration::from_secs(5)), async {
|
||||
let slot = self.bucket_fence_registry.reservation_slot(bucket, incarnation);
|
||||
let mut cached = slot.owner.lock().await;
|
||||
let owner = match cached.upgrade().filter(|owner| owner.is_reusable()) {
|
||||
Some(owner) => owner,
|
||||
None => {
|
||||
// Bucket deletion takes publication before lifecycle and metadata.
|
||||
let lock = self
|
||||
.handle_new_ns_lock(bucket, rustfs_common::table_catalog::TABLE_BUCKET_PUBLICATION_LOCK_PATH)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let publication = lock
|
||||
.get_read_lock(crate::set_disk::get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
// Older peers may acquire lifecycle without publication.
|
||||
let lifecycle = self
|
||||
.acquire_bucket_incarnation_fence(bucket, incarnation)
|
||||
.await
|
||||
.map_err(|error| DiskError::other(error.to_string()))?;
|
||||
let owner = Arc::new(super::bucket_fence::ReservationFenceGuard::new(publication, lifecycle, slot.clone()));
|
||||
if !owner.is_reusable() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
*cached = Arc::downgrade(&owner);
|
||||
owner
|
||||
}
|
||||
};
|
||||
drop(cached);
|
||||
self.reserve_local_object_metadata_with_owner(disk_ref, bucket, object, timeout, Some(owner))
|
||||
.await
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn reserve_local_object_metadata_with_owner(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
timeout: std::time::Duration,
|
||||
owner: Option<Arc<dyn crate::disk::os::MetadataReservationFence>>,
|
||||
) -> DiskResult<Uuid> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
wrapper.prepare_object_metadata_with_owner(bucket, object, owner),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
/// Refresh an inert token on its original disk and object.
|
||||
pub async fn renew_local_object_metadata_reservation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
wrapper.renew_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Atomically enqueue a prepared token on its original disk and object.
|
||||
pub async fn activate_local_object_metadata_reservation(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::bucket::utils::check_valid_bucket_name(bucket).is_err() || is_meta_bucketname(bucket) || token.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
wrapper.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed token on the currently admitted disk.
|
||||
/// A token already claimed by a physical mutation cannot be cancelled.
|
||||
pub async fn cancel_local_object_metadata_reservation(&self, disk_ref: &str, token: Uuid) -> DiskResult<bool> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
Ok(wrapper.cancel_object_metadata_reservation(token))
|
||||
}
|
||||
|
||||
/// Publish an ordinary staged user-object write using a one-shot disk reservation.
|
||||
/// The token is bound to the admitted disk instance and destination. A claimed
|
||||
/// physical mutation continues to own its lease if the RPC waiter is cancelled.
|
||||
/// The caller must retain the bucket lifecycle fence until the physical tail
|
||||
/// finishes; the reservation does not identify a bucket incarnation.
|
||||
pub async fn rename_local_data_reserved(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
source: (&str, &str),
|
||||
fi: &FileInfo,
|
||||
destination: (&str, &str),
|
||||
token: Uuid,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
if token.is_nil()
|
||||
|| source.0 != crate::disk::RUSTFS_META_TMP_BUCKET
|
||||
|| crate::bucket::utils::check_valid_bucket_name(destination.0).is_err()
|
||||
|| is_meta_bucketname(destination.0)
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(
|
||||
&fi.metadata,
|
||||
crate::disk::QUOTA_MUTATION_FENCE_METADATA_SUFFIX,
|
||||
)
|
||||
{
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
fi,
|
||||
destination,
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Resolve a quota lease against this request's captured storage instance.
|
||||
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
@@ -454,7 +270,6 @@ impl ECStore {
|
||||
scanner_publication_lease_token: scanner_token,
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -536,7 +351,7 @@ fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn admit_local_disk(
|
||||
async fn admit_local_disk(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk: &DiskStore,
|
||||
disk_id: Option<Uuid>,
|
||||
@@ -968,116 +783,6 @@ mod tests {
|
||||
.expect("seeded metadata bytes")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_target_rename_is_bound_to_the_admitted_disk() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let dir = tempfile::tempdir().expect("disk root");
|
||||
let id = Uuid::new_v4();
|
||||
let disk = target_disk(&ctx, dir.path(), id).await;
|
||||
let disk_ref = disk.endpoint().to_string();
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let version = Uuid::new_v4();
|
||||
let fi = target_file_info("object", version, b"reserved");
|
||||
seed_target(&disk, crate::disk::RUSTFS_META_TMP_BUCKET, "staged", fi.clone()).await;
|
||||
tokio::fs::create_dir_all(dir.path().join("target-bucket"))
|
||||
.await
|
||||
.expect("user bucket");
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(
|
||||
&disk_ref,
|
||||
"target-bucket/../.rustfs.sys",
|
||||
"object",
|
||||
std::time::Duration::from_secs(5),
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
assert!(matches!(
|
||||
store
|
||||
.reserve_local_object_metadata(&disk_ref, "", ".rustfs.sys/config", std::time::Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
let token = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve current disk");
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(&disk_ref, ("target-bucket", "staged"), &fi, ("target-bucket", "object"), token)
|
||||
.await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
store
|
||||
.activate_local_object_metadata_reservation(&disk_ref, "target-bucket", "object", token)
|
||||
.await
|
||||
.expect("activate token");
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
token,
|
||||
)
|
||||
.await
|
||||
.expect("reserved publication");
|
||||
assert!(
|
||||
!store
|
||||
.cancel_local_object_metadata_reservation(&disk_ref, token)
|
||||
.await
|
||||
.expect("consumed token")
|
||||
);
|
||||
let read = disk
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("read reserved publication");
|
||||
assert_eq!(read.data, fi.data);
|
||||
|
||||
let stale = store
|
||||
.reserve_local_object_metadata(&disk_ref, "target-bucket", "object", std::time::Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve before disk replacement");
|
||||
let replacement = target_disk(&ctx, dir.path(), id).await;
|
||||
assert!(matches!(
|
||||
store
|
||||
.rename_local_data_reserved(
|
||||
&disk_ref,
|
||||
(crate::disk::RUSTFS_META_TMP_BUCKET, "staged"),
|
||||
&fi,
|
||||
("target-bucket", "object"),
|
||||
stale,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::DiskOngoingReq)
|
||||
));
|
||||
let after = replacement
|
||||
.read_version(
|
||||
"target-bucket",
|
||||
"target-bucket",
|
||||
"object",
|
||||
&version.to_string(),
|
||||
&crate::disk::ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("replacement retains publication");
|
||||
assert_eq!(after.data, fi.data);
|
||||
assert!(store.cancel_local_object_metadata_reservation(&disk_ref, stale).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_lookup_binds_real_disk_and_owner_to_one_instance() {
|
||||
for warm in [false, true] {
|
||||
|
||||
@@ -925,6 +925,13 @@ impl MetaCacheEntriesSorted {
|
||||
|
||||
const METACACHE_STREAM_VERSION: u8 = 2;
|
||||
const METACACHE_READ_CHUNK_SIZE: usize = 64 << 10;
|
||||
/// Upper bound on one incremental reservation while accumulating a single
|
||||
/// metacache element. Read rounds double from [`METACACHE_READ_CHUNK_SIZE`]
|
||||
/// up to this bound so a legitimately large xl.meta does not pay one small
|
||||
/// read per chunk, while a corrupt length prefix still cannot force a
|
||||
/// reservation larger than the payload that has already arrived (see
|
||||
/// rustfs/rustfs#2715 and rustfs/rustfs#9120).
|
||||
const METACACHE_READ_MAX_CHUNK_SIZE: usize = 4 << 20;
|
||||
/// msgpack `false`: the entry-stream terminator, and the only byte a legacy
|
||||
/// reader accepts after it (see `MetacacheWriter::close_with_limit_reached`).
|
||||
const METACACHE_TERMINATOR: u8 = 0xc2;
|
||||
@@ -1157,13 +1164,14 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
// A whole xl.meta can legitimately exceed the limit on its individual
|
||||
// msgpack values. Grow only as payload arrives, so a corrupt length
|
||||
// prefix cannot cause a speculative allocation of the advertised size.
|
||||
let mut chunk = METACACHE_READ_CHUNK_SIZE;
|
||||
while self.buf.len() < read_size {
|
||||
let start = self.buf.len();
|
||||
let extra = if lookahead {
|
||||
let min_chunk = METACACHE_READ_CHUNK_SIZE.saturating_sub(start);
|
||||
(read_size - start).clamp(min_chunk, METACACHE_READ_CHUNK_SIZE)
|
||||
(read_size - start).clamp(min_chunk, chunk)
|
||||
} else {
|
||||
(read_size - start).min(METACACHE_READ_CHUNK_SIZE)
|
||||
(read_size - start).min(chunk)
|
||||
};
|
||||
if let Err(e) = self.buf.try_reserve(extra) {
|
||||
let err = Error::other(format!("metacache stream: buffer allocation of {extra} bytes failed: {e}"));
|
||||
@@ -1194,6 +1202,7 @@ impl<R: AsyncRead + Unpin> MetacacheReader<R> {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
chunk = chunk.saturating_mul(2).min(METACACHE_READ_MAX_CHUNK_SIZE);
|
||||
}
|
||||
|
||||
self.pos = read_size;
|
||||
@@ -2155,6 +2164,76 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// rustfs/rustfs#9120: a listing stream made of many ordinary entries must
|
||||
/// decode completely even when the aggregate stream exceeds the per-value
|
||||
/// `MAX_MSGP_ELEMENT_SIZE` bound that release 1.0.4 applied to every
|
||||
/// element.
|
||||
#[tokio::test]
|
||||
async fn test_reader_wide_listing_aggregate_exceeds_per_value_limit() {
|
||||
// ~2.9 KiB of metadata per entry matches the per-entry cost observed in
|
||||
// the #9120 production report; 6,000 entries push the stream past 16 MiB.
|
||||
const ENTRIES: usize = 6_000;
|
||||
let metadata = vec![0xab; 2_880];
|
||||
let entries: Vec<MetaCacheEntry> = (0..ENTRIES)
|
||||
.map(|i| MetaCacheEntry {
|
||||
name: format!("bigdir/object-{i:05}"),
|
||||
metadata: metadata.clone(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
let mut writer = MetacacheWriter::new(&mut output);
|
||||
writer.write(&entries).await.expect("wide listing must encode");
|
||||
writer.close().await.expect("stream must close");
|
||||
let bytes = output.into_inner();
|
||||
assert!(
|
||||
bytes.len() > 16 << 20,
|
||||
"fixture must exceed the per-value limit, got {} bytes",
|
||||
bytes.len()
|
||||
);
|
||||
let mut reader = MetacacheReader::new(Cursor::new(bytes));
|
||||
let decoded = reader.read_all().await.expect("wide listing must decode");
|
||||
assert_eq!(decoded, entries);
|
||||
}
|
||||
|
||||
/// The #9120 failure shape seen in production: one entry whose metadata is
|
||||
/// an xl.meta that grew past 16 MiB with its version history, listed among
|
||||
/// ordinary neighbors and skipped past without losing the framing.
|
||||
#[tokio::test]
|
||||
async fn test_reader_oversized_entry_among_ordinary_neighbors_stays_listable() {
|
||||
let small = |i: u8| MetaCacheEntry {
|
||||
name: format!("bigdir/object-{i}"),
|
||||
metadata: vec![i; 64],
|
||||
..Default::default()
|
||||
};
|
||||
let oversized = MetaCacheEntry {
|
||||
name: "bigdir/huge-versions".to_string(),
|
||||
metadata: vec![0xcd; (16 << 20) + 12_345],
|
||||
..Default::default()
|
||||
};
|
||||
let entries = vec![small(0), small(1), oversized.clone(), small(2), small(3)];
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
let mut writer = MetacacheWriter::new(&mut output);
|
||||
writer.write(&entries).await.expect("mixed listing must encode");
|
||||
writer.close().await.expect("stream must close");
|
||||
|
||||
let mut reader = MetacacheReader::new(Cursor::new(output.into_inner()));
|
||||
assert_eq!(
|
||||
reader.peek().await.expect("first entry").expect("present"),
|
||||
entries[0],
|
||||
"an ordinary entry before the oversized one must decode"
|
||||
);
|
||||
reader.skip(1).await.expect("skip the first entry");
|
||||
assert_eq!(reader.peek().await.expect("second entry").expect("present"), entries[1]);
|
||||
assert_eq!(
|
||||
reader.peek().await.expect("oversized entry").expect("present"),
|
||||
oversized,
|
||||
"an entry past the per-value limit must round-trip whole"
|
||||
);
|
||||
reader.skip(1).await.expect("skipping the oversized entry must stay aligned");
|
||||
assert_eq!(reader.read_all().await.expect("tail must decode"), entries[3..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_info_versions_with_free_versions_includes_persisted_tier_cleanup_records() {
|
||||
let version_id = Uuid::new_v4();
|
||||
@@ -3775,6 +3854,74 @@ mod tests {
|
||||
assert!(corrupt_crc_entry.is_latest_delete_marker());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_and_list_select_same_non_free_version_after_cleanup_record() {
|
||||
let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
|
||||
.expect("load realistic xl.meta");
|
||||
let object = fm.versions[0].into_fileinfo("bucket", "object", true).expect("decode object");
|
||||
let mut free_meta_sys = HashMap::new();
|
||||
rustfs_utils::http::insert_bytes(&mut free_meta_sys, rustfs_utils::http::SUFFIX_FREE_VERSION, Vec::new());
|
||||
let free = FileMetaShallowVersion::try_from(FileMetaVersion {
|
||||
version_type: VersionType::Delete,
|
||||
delete_marker: Some(MetaDeleteMarker {
|
||||
version_id: Some(Uuid::from_u128(42)),
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(2_000_000_000).expect("free timestamp")),
|
||||
meta_sys: free_meta_sys,
|
||||
}),
|
||||
write_version: 1,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("encode cleanup record");
|
||||
fm.versions.insert(0, free);
|
||||
let entry = MetaCacheEntry {
|
||||
name: "object".to_string(),
|
||||
metadata: fm.marshal_msg().expect("encode metadata"),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let scanned = fm
|
||||
.get_file_info_versions("bucket", "object", false)
|
||||
.expect("scanner version decode");
|
||||
assert_eq!(scanned.free_versions.len(), 1);
|
||||
assert_eq!(scanned.versions.len(), 1);
|
||||
assert!(!scanned.versions[0].is_latest, "free record occupies the original first position");
|
||||
assert!(!entry.clone().is_latest_delete_marker(), "cleanup record is not a visible delete marker");
|
||||
let listed = entry.to_fileinfo("bucket").expect("LIST version decode");
|
||||
assert_eq!(listed.version_id, object.version_id);
|
||||
assert_eq!(listed.version_id, scanned.versions[0].version_id);
|
||||
assert_eq!(listed.mod_time, scanned.versions[0].mod_time);
|
||||
assert_eq!(listed.size, scanned.versions[0].size);
|
||||
assert!(!listed.deleted);
|
||||
|
||||
let marker_id = Uuid::from_u128(43);
|
||||
fm.add_version(FileInfo {
|
||||
volume: "bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(marker_id),
|
||||
deleted: true,
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(2_100_000_000).expect("marker timestamp")),
|
||||
..Default::default()
|
||||
})
|
||||
.expect("add current delete marker");
|
||||
let mut marker_entry = MetaCacheEntry {
|
||||
name: "object".to_string(),
|
||||
metadata: fm.marshal_msg().expect("encode marker metadata"),
|
||||
cached: None,
|
||||
reusable: false,
|
||||
};
|
||||
let scanned = fm
|
||||
.get_file_info_versions("bucket", "object", false)
|
||||
.expect("scan marker stack");
|
||||
assert_eq!(scanned.versions.len(), 2);
|
||||
assert_eq!(scanned.versions[0].version_id, Some(marker_id));
|
||||
assert!(scanned.versions[0].deleted);
|
||||
assert!(!scanned.versions[1].deleted, "older object must remain hidden by the current marker");
|
||||
assert!(marker_entry.is_latest_delete_marker());
|
||||
let listed = marker_entry.to_fileinfo("bucket").expect("LIST marker version");
|
||||
assert_eq!(listed.version_id, scanned.versions[0].version_id);
|
||||
assert!(listed.deleted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_info_versions_and_borrowed_slice_decode_match_cached_and_full_decoders() {
|
||||
let mut fm = FileMeta::load(&crate::test_data::create_realistic_object_xlmeta().expect("realistic xl.meta"))
|
||||
|
||||
@@ -128,6 +128,20 @@ impl Error {
|
||||
Error::TransientSkip { message: message.into() }
|
||||
}
|
||||
|
||||
/// S3 `SlowDown` ("Please reduce your request rate") is a rate signal.
|
||||
/// Callers must cut their own concurrency instead of retrying at full speed.
|
||||
pub(crate) fn is_slow_down(&self) -> bool {
|
||||
match self {
|
||||
Error::Storage(error) => matches!(error, EcstoreError::SlowDown | EcstoreError::TooManyOpenFiles),
|
||||
Error::Disk(error) => matches!(error, DiskError::TooManyOpenFiles),
|
||||
Error::TransientSkip { message } | Error::Other(message) | Error::TaskExecutionFailed { message } => {
|
||||
message.contains("Please reduce your request rate")
|
||||
}
|
||||
Error::Io(error) => error.kind() == std::io::ErrorKind::WouldBlock,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a heal operation can be retried without changing its inputs.
|
||||
pub(crate) fn is_recoverable_heal(&self) -> bool {
|
||||
match classify_heal_error(self) {
|
||||
@@ -449,6 +463,22 @@ mod tests {
|
||||
use super::Error;
|
||||
use crate::heal::{DiskError, EcstoreError};
|
||||
|
||||
#[test]
|
||||
fn slow_down_is_a_rate_signal_across_wrapped_shapes() {
|
||||
assert!(Error::Storage(EcstoreError::SlowDown).is_slow_down());
|
||||
assert!(Error::Storage(EcstoreError::TooManyOpenFiles).is_slow_down());
|
||||
assert!(Error::Disk(DiskError::TooManyOpenFiles).is_slow_down());
|
||||
assert!(Error::transient_skip("Please reduce your request rate").is_slow_down());
|
||||
assert!(
|
||||
Error::TaskExecutionFailed {
|
||||
message: "Failed to recreate missing object b/o: Storage error: Please reduce your request rate".to_owned(),
|
||||
}
|
||||
.is_slow_down()
|
||||
);
|
||||
assert!(!Error::Storage(EcstoreError::FileNotFound).is_slow_down());
|
||||
assert!(!Error::transient_skip("DiskNotFound").is_slow_down());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_metadata_failures_keep_their_typed_recovery_classification() {
|
||||
use crate::heal::storage_api::owner::{EcstorePoolMetadataError, EcstorePoolMetadataFailure};
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use crate::error::{HealErrorClass, classify_heal_error};
|
||||
use crate::heal::{
|
||||
pacing::HealPageRate,
|
||||
progress::{HealProgress, add_bytes, increment_counter},
|
||||
resume::{
|
||||
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ReplacementTargetIdentity, ResumeManager,
|
||||
@@ -27,7 +28,6 @@ use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use metrics::{counter, gauge};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
@@ -129,12 +129,12 @@ pub struct ErasureSetHealer {
|
||||
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
|
||||
replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
|
||||
work_parent: Option<super::manager::HealWork>,
|
||||
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
||||
mainline_pacer: Option<super::pacing::ForegroundPacing>,
|
||||
}
|
||||
|
||||
async fn acquire_page_permit(
|
||||
semaphore: Arc<Semaphore>,
|
||||
pacer: Option<&super::pacing::MainlinePacer>,
|
||||
rate: &HealPageRate,
|
||||
cancel: &tokio_util::sync::CancellationToken,
|
||||
) -> Result<tokio::sync::OwnedSemaphorePermit> {
|
||||
let acquire = || async {
|
||||
@@ -147,18 +147,16 @@ async fn acquire_page_permit(
|
||||
let mut paid_pause = false;
|
||||
loop {
|
||||
let permit = acquire().await?;
|
||||
if let Some(pacer) = pacer {
|
||||
// Keep the real permit on the low-pressure path. Every acquisition
|
||||
// gets a fresh decision, including a waiter that queued a second
|
||||
// time. One completed pause is a bounded minimum-progress grant.
|
||||
match pacer.admission_decision() {
|
||||
super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
|
||||
drop(permit);
|
||||
paid_pause = pacer.wait_after_admission(cancel, pressure).await?;
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
// Keep the real permit on the low-pressure path. Every acquisition
|
||||
// gets a fresh decision, including a waiter that queued a second
|
||||
// time. One completed pause is a bounded minimum-progress grant.
|
||||
match rate.admission_decision() {
|
||||
super::pacing::PacingDecision::Wait(pressure) if !paid_pause => {
|
||||
drop(permit);
|
||||
paid_pause = rate.wait_after_admission(cancel, pressure).await?;
|
||||
continue;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
return Ok(permit);
|
||||
}
|
||||
@@ -167,7 +165,7 @@ async fn acquire_page_permit(
|
||||
#[cfg(test)]
|
||||
mod mainline_pacing_tests {
|
||||
use super::*;
|
||||
use crate::heal::pacing::{MainlinePacer, TestPressure};
|
||||
use crate::heal::pacing::{ForegroundPacing, HealPageRate, TestPressure};
|
||||
use rustfs_concurrency::WorkloadClass;
|
||||
use std::sync::atomic::Ordering;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -181,15 +179,15 @@ mod mainline_pacing_tests {
|
||||
.await
|
||||
.expect("existing object owns capacity");
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
|
||||
let pacer = Arc::new(
|
||||
MainlinePacer::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer"),
|
||||
);
|
||||
let pacer =
|
||||
ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_millis(250), HealRequestSource::Admin).expect("pacer");
|
||||
let rate = Arc::new(HealPageRate::new(1, Some(pacer)));
|
||||
let cancel = CancellationToken::new();
|
||||
let waiting = tokio::spawn({
|
||||
let semaphore = semaphore.clone();
|
||||
let pacer = pacer.clone();
|
||||
let rate = rate.clone();
|
||||
let cancel = cancel.clone();
|
||||
async move { acquire_page_permit(semaphore, Some(&pacer), &cancel).await }
|
||||
async move { acquire_page_permit(semaphore, &rate, &cancel).await }
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
provider.active.store(100, Ordering::SeqCst);
|
||||
@@ -201,12 +199,13 @@ mod mainline_pacing_tests {
|
||||
assert_eq!(semaphore.available_permits(), 1);
|
||||
let deadline = tokio::time::timeout(
|
||||
Duration::from_millis(10),
|
||||
acquire_page_permit(semaphore.clone(), Some(&pacer), &CancellationToken::new()),
|
||||
acquire_page_permit(semaphore.clone(), &rate, &CancellationToken::new()),
|
||||
)
|
||||
.await;
|
||||
assert!(deadline.is_err());
|
||||
assert_eq!(semaphore.available_permits(), 1, "deadline must release all permits");
|
||||
let permit = acquire_page_permit(semaphore.clone(), None, &CancellationToken::new())
|
||||
let unpaced = HealPageRate::new(1, None);
|
||||
let permit = acquire_page_permit(semaphore.clone(), &unpaced, &CancellationToken::new())
|
||||
.await
|
||||
.expect("unpaced admission");
|
||||
assert_eq!(semaphore.available_permits(), 0, "disabling pacing cannot disable the hard cap");
|
||||
@@ -222,10 +221,11 @@ mod mainline_pacing_tests {
|
||||
let occupied = semaphore.clone().acquire_owned().await.expect("queue both waiters");
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 0));
|
||||
let pause = Duration::from_millis(250);
|
||||
let pacer = MainlinePacer::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
|
||||
let pacer = ForegroundPacing::new(provider.clone(), 80, 80, pause, HealRequestSource::Admin).expect("pacer");
|
||||
let rate = HealPageRate::new(1, Some(pacer));
|
||||
let cancel = CancellationToken::new();
|
||||
let mut first = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
|
||||
let mut second = Box::pin(acquire_page_permit(semaphore.clone(), Some(&pacer), &cancel));
|
||||
let mut first = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
|
||||
let mut second = Box::pin(acquire_page_permit(semaphore.clone(), &rate, &cancel));
|
||||
assert!(futures::poll!(first.as_mut()).is_pending());
|
||||
assert!(futures::poll!(second.as_mut()).is_pending());
|
||||
drop(occupied);
|
||||
@@ -330,6 +330,20 @@ impl ErasureSetHealer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Page fan-out for one whole erasure-set pass (stable across buckets).
|
||||
fn pass_page_concurrency_limit(&self) -> usize {
|
||||
if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
|
||||
// Versions of the same object must not race without its namespace lock.
|
||||
1
|
||||
} else {
|
||||
Self::effective_heal_page_object_concurrency_for_source(
|
||||
self.source,
|
||||
self.heal_opts.scan_mode,
|
||||
self.replacement_task_id.is_some(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Map the shared heal error class onto this healer's per-object actions.
|
||||
///
|
||||
/// `classify_heal_error` is the only classifier; this adapter decides what
|
||||
@@ -374,7 +388,7 @@ impl ErasureSetHealer {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
|
||||
self.mainline_pacer = pacer;
|
||||
self
|
||||
}
|
||||
@@ -465,12 +479,7 @@ impl ErasureSetHealer {
|
||||
None
|
||||
};
|
||||
let buckets = admin_buckets.as_deref().unwrap_or(buckets);
|
||||
if checkpoint_manager
|
||||
.get_checkpoint()
|
||||
.await
|
||||
.admin
|
||||
.is_some_and(|admin| admin.completed)
|
||||
{
|
||||
if checkpoint_manager.admin_completed().await {
|
||||
self.initialize_progress(buckets, &resume_manager.get_state().await).await;
|
||||
return Ok(());
|
||||
}
|
||||
@@ -491,14 +500,7 @@ impl ErasureSetHealer {
|
||||
}
|
||||
|
||||
if self.admin_task.is_some() {
|
||||
let mut admin = checkpoint_manager
|
||||
.get_checkpoint()
|
||||
.await
|
||||
.admin
|
||||
.ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
|
||||
admin.completed = true;
|
||||
admin.outcome.finish(None);
|
||||
checkpoint_manager.set_admin(admin).await?;
|
||||
checkpoint_manager.complete_admin().await?;
|
||||
} else {
|
||||
checkpoint_manager.cleanup().await?;
|
||||
resume_manager.cleanup().await?;
|
||||
@@ -719,7 +721,10 @@ impl ErasureSetHealer {
|
||||
}
|
||||
return Err(Error::DanglingDeleteDeferred { retry_not_before });
|
||||
}
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
// One rate controller for the whole pass: SlowDown cool-down and page
|
||||
// fan-out persist across buckets instead of resetting at each one.
|
||||
let page_rate = Arc::new(HealPageRate::new(self.pass_page_concurrency_limit(), self.mainline_pacer.clone()));
|
||||
let (current_bucket_index, current_object_index) = checkpoint_manager.resume_position().await;
|
||||
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -727,14 +732,17 @@ impl ErasureSetHealer {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE_HEALER,
|
||||
set_disk_id,
|
||||
current_bucket_index = checkpoint.current_bucket_index,
|
||||
current_object_index = checkpoint.current_object_index,
|
||||
current_bucket_index,
|
||||
current_object_index,
|
||||
state = "resuming",
|
||||
"Erasure set resumed"
|
||||
);
|
||||
|
||||
// 2. initialize progress
|
||||
self.initialize_progress(buckets, &state).await;
|
||||
// Snapshot before any baseline write: set_progress_baseline overwrites
|
||||
// the saved generation, which would hide a real generation mismatch.
|
||||
let checkpoint = checkpoint_manager.progress_snapshot().await;
|
||||
let (baseline_known, baseline_count, baseline_size, baseline_generation) = {
|
||||
let baseline = self.progress.read().await;
|
||||
(
|
||||
@@ -761,17 +769,7 @@ impl ErasureSetHealer {
|
||||
let mut successful_objects = state.successful_objects;
|
||||
let mut failed_objects = state.failed_objects;
|
||||
let mut skipped_objects = state.skipped_objects;
|
||||
let checkpoint_has_progress = checkpoint.baseline_known
|
||||
|| checkpoint.successful_objects > 0
|
||||
|| checkpoint.failed_object_count > 0
|
||||
|| checkpoint.skipped_object_count > 0
|
||||
|| checkpoint.skipped_new_versions > 0
|
||||
|| checkpoint.skipped_ilm_expired > 0
|
||||
|| checkpoint.processed_bytes > 0
|
||||
|| checkpoint.total_objects > 0
|
||||
|| checkpoint.total_bytes > 0
|
||||
|| checkpoint.baseline_generation.is_some()
|
||||
|| checkpoint.counter_unknown;
|
||||
let checkpoint_has_progress = checkpoint.has_progress();
|
||||
let checkpoint_generation_mismatch = checkpoint.baseline_known && checkpoint.baseline_generation != baseline_generation;
|
||||
let mut restored_counter_unknown = state.counter_unknown || checkpoint.counter_unknown;
|
||||
if checkpoint_has_progress {
|
||||
@@ -855,6 +853,7 @@ impl ErasureSetHealer {
|
||||
resume_manager,
|
||||
checkpoint_manager,
|
||||
state.start_time,
|
||||
&page_rate,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -1021,14 +1020,14 @@ impl ErasureSetHealer {
|
||||
if failed_objects > 0 || skipped_objects > 0 || failed_buckets > 0 {
|
||||
if self.replacement_task_id.is_some() {
|
||||
let state = resume_manager.get_state().await;
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
let (grace_objects, retry_not_before, counter_unknown) = checkpoint_manager.dangling_delete_grace().await;
|
||||
if state.can_retry()
|
||||
&& failed_objects == 0
|
||||
&& failed_buckets == 0
|
||||
&& skipped_objects > 0
|
||||
&& skipped_objects == checkpoint.dangling_delete_grace_objects
|
||||
&& !checkpoint.counter_unknown
|
||||
&& let Some(retry_not_before) = checkpoint.dangling_delete_retry_not_before
|
||||
&& skipped_objects == grace_objects
|
||||
&& !counter_unknown
|
||||
&& let Some(retry_not_before) = retry_not_before
|
||||
{
|
||||
// Publish the generation's rewind and wait first. If the
|
||||
// checkpoint reset is interrupted, initialization reapplies
|
||||
@@ -1168,14 +1167,9 @@ impl ErasureSetHealer {
|
||||
|
||||
let object_key = format!("{RUSTFS_META_BUCKET}/{POOL_META_NAME}");
|
||||
let checkpoint_key = compose_key(&object_key, None);
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
if checkpoint.processed_objects.contains(&checkpoint_key)
|
||||
|| checkpoint.failed_objects.contains(&checkpoint_key)
|
||||
|| checkpoint.skipped_objects.contains(&checkpoint_key)
|
||||
{
|
||||
if checkpoint_manager.is_object_settled(&checkpoint_key).await {
|
||||
return Ok(());
|
||||
}
|
||||
drop(checkpoint);
|
||||
|
||||
self.verify_replacement_identity_fence("pool metadata").await?;
|
||||
resume_manager
|
||||
@@ -1429,7 +1423,7 @@ impl ErasureSetHealer {
|
||||
|
||||
/// Heal one bucket's ordinary objects and report unresolved delete recovery separately.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager), fields(bucket = %bucket, bucket_index = bucket_index))]
|
||||
#[tracing::instrument(skip(self, current_object_index, processed_objects, successful_objects, failed_objects, skipped_objects, resume_manager, checkpoint_manager, page_rate), fields(bucket = %bucket, bucket_index = bucket_index))]
|
||||
async fn heal_bucket_with_resume(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -1443,6 +1437,7 @@ impl ErasureSetHealer {
|
||||
resume_manager: &ResumeManager,
|
||||
checkpoint_manager: &CheckpointManager,
|
||||
started_at_secs: u64,
|
||||
page_rate: &Arc<HealPageRate>,
|
||||
) -> Result<bool> {
|
||||
debug!(
|
||||
target: "rustfs::heal::erasure_healer",
|
||||
@@ -1492,17 +1487,6 @@ impl ErasureSetHealer {
|
||||
// detects a non-advancing cursor even though next_marker embeds a
|
||||
// transient cache id that changes between identical pages.
|
||||
let mut previous_page_last: Option<(String, Option<String>)> = None;
|
||||
let page_concurrency_limit =
|
||||
if self.source == HealRequestSource::AutoHeal && self.replacement_task_id.is_some() && self.heal_opts.no_lock {
|
||||
// Versions of the same object must not race without its namespace lock.
|
||||
1
|
||||
} else {
|
||||
Self::effective_heal_page_object_concurrency_for_source(
|
||||
self.source,
|
||||
self.heal_opts.scan_mode,
|
||||
self.replacement_task_id.is_some(),
|
||||
)
|
||||
};
|
||||
let in_flight = Arc::new(AtomicUsize::new(0));
|
||||
// Per-bucket sample caps for per-object warn! lines: a flapping rebuild
|
||||
// disk can fail/skip hundreds of thousands of versions in one sweep, so
|
||||
@@ -1524,9 +1508,7 @@ impl ErasureSetHealer {
|
||||
let include_lifecycle_object_info = lifecycle_expiry_context.is_some();
|
||||
|
||||
loop {
|
||||
if let Some(pacer) = &self.mainline_pacer {
|
||||
pacer.wait(&self.cancel_token).await?;
|
||||
}
|
||||
page_rate.wait_before_page(&self.cancel_token).await?;
|
||||
self.verify_replacement_identity_fence("page scan").await?;
|
||||
if let Some(expected) = incarnation {
|
||||
self.storage.validate_bucket_incarnation(bucket, Some(expected)).await?;
|
||||
@@ -1548,10 +1530,8 @@ impl ErasureSetHealer {
|
||||
.await?
|
||||
};
|
||||
let page_is_empty = objects.is_empty();
|
||||
let checkpoint = checkpoint_manager.get_checkpoint().await;
|
||||
let page_resume_index = *current_object_index;
|
||||
let semaphore = Arc::new(Semaphore::new(page_concurrency_limit));
|
||||
let mut object_locks: HashMap<String, Arc<tokio::sync::Mutex<()>>> = HashMap::new();
|
||||
let semaphore = Arc::new(Semaphore::new(page_rate.effective_concurrency()));
|
||||
let mut page_tasks = FuturesUnordered::new();
|
||||
let mut completed_in_page = 0usize;
|
||||
|
||||
@@ -1565,10 +1545,7 @@ impl ErasureSetHealer {
|
||||
|
||||
// Per-version dedup identity — the single canonical key.
|
||||
let key = compose_key(&item.name, item.version_id.as_deref());
|
||||
if checkpoint.processed_objects.contains(&key)
|
||||
|| checkpoint.failed_objects.contains(&key)
|
||||
|| checkpoint.skipped_objects.contains(&key)
|
||||
{
|
||||
if checkpoint_manager.is_object_settled(&key).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1723,11 +1700,10 @@ impl ErasureSetHealer {
|
||||
let set_label = set_disk_id.to_string();
|
||||
let heal_opts = self.heal_opts;
|
||||
let semaphore = semaphore.clone();
|
||||
let object_lock = object_locks.entry(object_name.clone()).or_default().clone();
|
||||
let target_endpoints = self.target_endpoints.clone();
|
||||
let replacement_commit_evidence_required = self.replacement_task_id.is_some();
|
||||
let retry_replacement = replacement_commit_evidence_required && self.source == HealRequestSource::AutoHeal;
|
||||
let mainline_pacer = self.mainline_pacer.clone();
|
||||
let page_rate = page_rate.clone();
|
||||
|
||||
let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
|
||||
let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
|
||||
@@ -1738,22 +1714,16 @@ impl ErasureSetHealer {
|
||||
.as_ref()
|
||||
.map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
|
||||
let work = async move {
|
||||
// Versions share one namespace lock. Queue them before taking a
|
||||
// page permit so a waiting version cannot block other objects.
|
||||
let _object_guard = tokio::select! {
|
||||
biased;
|
||||
_ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
|
||||
guard = object_lock.lock() => guard,
|
||||
};
|
||||
let mut retries = 0;
|
||||
let mut saw_slow_down = false;
|
||||
let mut admin_identity = admin_identity;
|
||||
let mut admin_task_id = admin_task_id;
|
||||
loop {
|
||||
let permit = acquire_page_permit(semaphore.clone(), mainline_pacer.as_deref(), &cancel_token).await;
|
||||
let permit = acquire_page_permit(semaphore.clone(), &page_rate, &cancel_token).await;
|
||||
|
||||
let permit = match permit {
|
||||
Ok(permit) => permit,
|
||||
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err)), None),
|
||||
Err(err) => return (dedup_key, object_name, version_id, (0, Err(err), saw_slow_down), None),
|
||||
};
|
||||
|
||||
let in_flight_guard = PageConcurrencyGuard::new(in_flight.clone(), set_label.clone());
|
||||
@@ -1761,7 +1731,8 @@ impl ErasureSetHealer {
|
||||
if let (Some(identity), Some(task_id)) = (admin_identity.take(), admin_task_id.take()) {
|
||||
let (result, canonical) =
|
||||
admin::heal_object(storage.as_ref(), &heal_opts, identity, &task_id, &cancel_token).await;
|
||||
return (dedup_key, object_name, version_id, result, canonical);
|
||||
let saw_slow_down = saw_slow_down || matches!(&result.1, Err(err) if err.is_slow_down());
|
||||
return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), canonical);
|
||||
}
|
||||
|
||||
// Always go through heal_object. Genuine absence flows through
|
||||
@@ -1863,18 +1834,23 @@ impl ErasureSetHealer {
|
||||
|
||||
drop(in_flight_guard);
|
||||
drop(permit);
|
||||
if matches!(&result.1, Err(err) if err.is_slow_down()) {
|
||||
saw_slow_down = true;
|
||||
}
|
||||
if retry_replacement
|
||||
&& retries < REPLACEMENT_OBJECT_MAX_RETRIES
|
||||
&& matches!(&result.1, Err(Error::TransientSkip { .. }))
|
||||
{
|
||||
retries += 1;
|
||||
tokio::select! {
|
||||
_ = cancel_token.cancelled() => return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled)), None),
|
||||
_ = cancel_token.cancelled() => {
|
||||
return (dedup_key, object_name, version_id, (0, Err(Error::TaskCancelled), saw_slow_down), None);
|
||||
}
|
||||
_ = tokio::time::sleep(Duration::from_millis(250 * u64::from(retries))) => {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
return (dedup_key, object_name, version_id, result, None);
|
||||
return (dedup_key, object_name, version_id, (result.0, result.1, saw_slow_down), None);
|
||||
}
|
||||
};
|
||||
page_tasks.push(async move {
|
||||
@@ -1885,7 +1861,7 @@ impl ErasureSetHealer {
|
||||
};
|
||||
match result {
|
||||
Ok(result) => result,
|
||||
Err(error) => (key, object, version, (0, Err(error)), None),
|
||||
Err(error) => (key, object, version, (0, Err(error), false), None),
|
||||
}
|
||||
} else {
|
||||
work.await
|
||||
@@ -1894,7 +1870,16 @@ impl ErasureSetHealer {
|
||||
}
|
||||
|
||||
while let Some((key, object, version_id, result, canonical)) = page_tasks.next().await {
|
||||
let (object_size, result) = result;
|
||||
let (object_size, result, saw_slow_down) = result;
|
||||
// Exactly one rate observation per finished object, including
|
||||
// SlowDown that later succeeded on a replacement retry.
|
||||
page_rate.observe_object_outcome(
|
||||
&match &result {
|
||||
Ok(_) => Ok(true),
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
saw_slow_down,
|
||||
);
|
||||
let mut telemetry_unknown = false;
|
||||
let checkpoint_outcome = match result {
|
||||
Ok(true) => {
|
||||
@@ -2492,6 +2477,7 @@ mod resume_loop_tests {
|
||||
use super::{
|
||||
ErasureSetHealer, NANOS_PER_SECOND, NEW_VERSION_SKIP_GRACE_SECS, should_skip_new_version, target_outcomes_complete,
|
||||
};
|
||||
use crate::heal::pacing::HealPageRate;
|
||||
use crate::heal::progress::HealProgress;
|
||||
use crate::heal::resume::{
|
||||
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, RESUME_CHECKPOINT_FILE,
|
||||
@@ -3073,6 +3059,7 @@ mod resume_loop_tests {
|
||||
let mut successful = 0u64;
|
||||
let mut failed = 0u64;
|
||||
let mut skipped = 0u64;
|
||||
let page_rate = Arc::new(HealPageRate::new(env.healer.pass_page_concurrency_limit(), None));
|
||||
let result = env
|
||||
.healer
|
||||
.heal_bucket_with_resume(
|
||||
@@ -3087,6 +3074,7 @@ mod resume_loop_tests {
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
state.start_time,
|
||||
&page_rate,
|
||||
)
|
||||
.await;
|
||||
(processed, successful, failed, skipped, result.map(|_| ()))
|
||||
@@ -3154,6 +3142,7 @@ mod resume_loop_tests {
|
||||
let mut skipped = 0;
|
||||
let started_at = env.resume.get_state().await.start_time;
|
||||
|
||||
let page_rate = Arc::new(HealPageRate::new(healer.pass_page_concurrency_limit(), None));
|
||||
let error = healer
|
||||
.heal_bucket_with_resume(
|
||||
"b",
|
||||
@@ -3167,6 +3156,7 @@ mod resume_loop_tests {
|
||||
&env.resume,
|
||||
&env.checkpoint,
|
||||
started_at,
|
||||
&page_rate,
|
||||
)
|
||||
.await
|
||||
.expect_err("a remounted target must not begin a new page scan");
|
||||
@@ -4057,64 +4047,6 @@ mod resume_loop_tests {
|
||||
assert_eq!(unique.len(), 50, "no version healed twice");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn page_serializes_versions_without_blocking_other_objects() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_HEAL_PAGE_PARALLEL_ENABLE, Some("true")),
|
||||
(rustfs_config::ENV_HEAL_PAGE_OBJECT_CONCURRENCY, Some("2")),
|
||||
],
|
||||
async {
|
||||
let mut env = make_env().await;
|
||||
env.healer.source = HealRequestSource::AutoHeal;
|
||||
env.healer = env
|
||||
.healer
|
||||
.with_replacement_targets(vec!["replacement-a".into()], Some(env.task_id.clone()));
|
||||
let gate = Arc::new(tokio::sync::Semaphore::new(0));
|
||||
*env.storage.heal_gate.lock().unwrap() = Some(gate.clone());
|
||||
for (name, version) in [("a", "v1"), ("a", "v2"), ("b", "v1")] {
|
||||
env.storage
|
||||
.set_result(name, Some(version), replacement_target_ok_result("replacement-a", name));
|
||||
}
|
||||
env.storage.set_page(
|
||||
None,
|
||||
Page {
|
||||
items: vec![
|
||||
item("a", Some("v1"), false),
|
||||
item("a", Some("v2"), false),
|
||||
item("b", Some("v1"), false),
|
||||
],
|
||||
next: None,
|
||||
truncated: false,
|
||||
},
|
||||
);
|
||||
let observe = async {
|
||||
let started = tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
loop {
|
||||
let notified = env.storage.heal_started.notified();
|
||||
if env.storage.calls().len() >= 2 {
|
||||
break;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok();
|
||||
let calls = env.storage.calls();
|
||||
gate.add_permits(3);
|
||||
assert!(started, "different objects should run concurrently");
|
||||
assert_eq!(calls.len(), 2);
|
||||
assert!(calls.iter().any(|(name, _)| name == "a"));
|
||||
assert!(calls.iter().any(|(name, _)| name == "b"));
|
||||
};
|
||||
let ((processed, successful, failed, skipped, result), ()) = tokio::join!(run(&env), observe);
|
||||
result.expect("all versions heal");
|
||||
assert_eq!((processed, successful, failed, skipped), (3, 3, 0, 0));
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn auto_replacement_runs_two_object_repairs_within_the_configured_limit() {
|
||||
temp_env::async_with_vars(
|
||||
|
||||
@@ -810,7 +810,14 @@ fn publish_heal_queue_length(queue: &PriorityHealQueue) {
|
||||
fn active_heal_for_dedup_key(active_heals: &HashMap<String, Arc<HealTask>>, key: &str) -> Option<(String, HealType)> {
|
||||
active_heals
|
||||
.iter()
|
||||
.find(|(_, task)| PriorityHealQueue::make_dedup_key_for_scope(&task.heal_type, &task.options) == key)
|
||||
.find(|(_, task)| {
|
||||
PriorityHealQueue::make_dedup_key_for_owner(
|
||||
&task.heal_type,
|
||||
&task.options,
|
||||
task.source,
|
||||
task.expected_mrf_bucket_incarnation_id,
|
||||
) == key
|
||||
})
|
||||
.map(|(task_id, task)| (task_id.clone(), task.heal_type.clone()))
|
||||
}
|
||||
|
||||
|
||||
@@ -611,7 +611,104 @@ impl PriorityHealQueue {
|
||||
|
||||
/// Create a deduplication key from a heal request
|
||||
pub(super) fn make_dedup_key(request: &HealRequest) -> String {
|
||||
Self::make_dedup_key_for_scope(&request.heal_type, &request.options)
|
||||
Self::make_dedup_key_for_owner(
|
||||
&request.heal_type,
|
||||
&request.options,
|
||||
request.source,
|
||||
request.expected_mrf_bucket_incarnation_id,
|
||||
)
|
||||
}
|
||||
|
||||
/// Bound MRF work can only merge with work for the same bucket lifetime.
|
||||
/// Length-prefix user-controlled components so a key cannot forge another
|
||||
/// version, scope, or incarnation through delimiter characters.
|
||||
pub(super) fn make_dedup_key_for_owner(
|
||||
heal_type: &HealType,
|
||||
options: &HealOptions,
|
||||
source: HealRequestSource,
|
||||
incarnation: Option<Uuid>,
|
||||
) -> String {
|
||||
if source == HealRequestSource::Mrf
|
||||
&& let Some(incarnation) = incarnation
|
||||
{
|
||||
use std::fmt::Write as _;
|
||||
|
||||
let (pool, set) = heal_scope_indices(heal_type, options);
|
||||
let mut key = format!("mrf-bound-v1:{incarnation}:{pool:?}:{set:?}:");
|
||||
let mut part = |value: &str| {
|
||||
// Formatting into a String is infallible.
|
||||
let _ = write!(key, "{}:{value}", value.len());
|
||||
};
|
||||
match heal_type {
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
part("object");
|
||||
part(bucket);
|
||||
part(object);
|
||||
match version_id {
|
||||
None => part("none"),
|
||||
Some(version_id) => {
|
||||
part("some");
|
||||
part(version_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
HealType::Metadata { bucket, object } => {
|
||||
part("metadata");
|
||||
part(bucket);
|
||||
part(object);
|
||||
}
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
} => {
|
||||
part("ecdecode");
|
||||
part(bucket);
|
||||
part(object);
|
||||
match version_id {
|
||||
None => part("none"),
|
||||
Some(version_id) => {
|
||||
part("some");
|
||||
part(version_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
HealType::DeleteMarkerPurge {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
purge,
|
||||
} => {
|
||||
part("delete-marker-purge");
|
||||
part(bucket);
|
||||
part(object);
|
||||
part(version_id);
|
||||
part(&purge.bucket_incarnation_id.to_string());
|
||||
part(&purge.marker_incarnation_id.to_string());
|
||||
part(&base64_simd::URL_SAFE_NO_PAD.encode_to_string(purge.marker_identity));
|
||||
}
|
||||
HealType::Bucket { bucket } => {
|
||||
part("bucket");
|
||||
part(bucket);
|
||||
}
|
||||
HealType::Prefix { bucket, prefix } => {
|
||||
part("prefix");
|
||||
part(bucket);
|
||||
part(prefix);
|
||||
}
|
||||
// Cluster and erasure-set work are not bucket-bound; preserve
|
||||
// their existing keys, including contains_erasure_set lookups.
|
||||
HealType::Cluster | HealType::ErasureSet { .. } => {
|
||||
return Self::make_dedup_key_for_scope(heal_type, options);
|
||||
}
|
||||
}
|
||||
return key;
|
||||
}
|
||||
Self::make_dedup_key_for_scope(heal_type, options)
|
||||
}
|
||||
|
||||
pub(super) fn make_dedup_key_for_scope(heal_type: &HealType, options: &HealOptions) -> String {
|
||||
|
||||
@@ -34,7 +34,7 @@ pub(super) fn running_mainline_pacer(
|
||||
request: &HealRequest,
|
||||
config: &HealConfig,
|
||||
provider: &Option<WorkloadSnapshotProviderRef>,
|
||||
) -> Option<Arc<crate::heal::pacing::MainlinePacer>> {
|
||||
) -> Option<crate::heal::pacing::ForegroundPacing> {
|
||||
let pace_running = request.source == HealRequestSource::Admin
|
||||
|| (request.source == HealRequestSource::AutoHeal
|
||||
&& matches!(&request.heal_type, HealType::ErasureSet { .. })
|
||||
@@ -43,14 +43,13 @@ pub(super) fn running_mainline_pacer(
|
||||
return None;
|
||||
}
|
||||
provider.as_ref().and_then(|provider| {
|
||||
crate::heal::pacing::MainlinePacer::new(
|
||||
crate::heal::pacing::ForegroundPacing::new(
|
||||
provider.clone(),
|
||||
config.mainline_read_utilization_high_percent,
|
||||
config.mainline_write_utilization_high_percent,
|
||||
config.mainline_max_sleep,
|
||||
request.source,
|
||||
)
|
||||
.map(Arc::new)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -117,6 +117,453 @@ async fn process_manager_queue_once(manager: &HealManager) {
|
||||
|
||||
struct MockStorage;
|
||||
|
||||
fn incarnation_bound_mrf_fallback(incarnation: Uuid) -> HealRequest {
|
||||
let mut request = HealRequest::object("mrf-recreated-bucket".to_owned(), "object".to_owned(), None);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
request.expected_mrf_bucket_incarnation_id = Some(incarnation);
|
||||
request.options.pool_index = Some(0);
|
||||
request.options.set_index = Some(0);
|
||||
request
|
||||
}
|
||||
|
||||
async fn assert_recreated_bucket_fallback_has_its_own_owner(existing_state: &str) {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::clone(&storage), None);
|
||||
let old_incarnation = Uuid::new_v4();
|
||||
let new_incarnation = Uuid::new_v4();
|
||||
let old_request = incarnation_bound_mrf_fallback(old_incarnation);
|
||||
let old_id = old_request.id.clone();
|
||||
match existing_state {
|
||||
"queued" => {
|
||||
let receipt = manager
|
||||
.submit_heal_request_with_receipt(old_request)
|
||||
.await
|
||||
.expect("old fallback must be admitted");
|
||||
assert_eq!(receipt.result, HealAdmissionResult::Accepted);
|
||||
}
|
||||
"active" => {
|
||||
manager
|
||||
.active_heals
|
||||
.lock()
|
||||
.await
|
||||
.insert(old_id.clone(), Arc::new(HealTask::from_request(old_request, Arc::clone(&storage))));
|
||||
}
|
||||
"retrying" => {
|
||||
manager.retrying_heals.lock().await.insert(
|
||||
old_id.clone(),
|
||||
RetryingHeal {
|
||||
request: old_request,
|
||||
error: "retry pending".to_owned(),
|
||||
cancel_token: CancellationToken::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
other => panic!("unknown fixture state: {other}"),
|
||||
}
|
||||
|
||||
let new_request = incarnation_bound_mrf_fallback(new_incarnation);
|
||||
let new_id = new_request.id.clone();
|
||||
assert_ne!(new_id, old_id, "the fallback must use a new request ID");
|
||||
let receipt = manager
|
||||
.submit_heal_request_with_receipt(new_request)
|
||||
.await
|
||||
.expect("new incarnation admission must be decided");
|
||||
assert_eq!(
|
||||
receipt.result,
|
||||
HealAdmissionResult::Accepted,
|
||||
"a {existing_state} fallback for an older bucket incarnation cannot own the new repair"
|
||||
);
|
||||
assert_eq!(receipt.task_id, new_id, "the new incarnation must retain its own task identity");
|
||||
let queue = manager.heal_queue.lock().await;
|
||||
assert!(
|
||||
queue
|
||||
.requests()
|
||||
.any(|request| { request.id == new_id && request.expected_mrf_bucket_incarnation_id == Some(new_incarnation) })
|
||||
);
|
||||
if existing_state == "queued" {
|
||||
assert!(
|
||||
queue
|
||||
.requests()
|
||||
.any(|request| { request.id == old_id && request.expected_mrf_bucket_incarnation_id == Some(old_incarnation) })
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_recreated_bucket_does_not_merge_into_queued_old_incarnation() {
|
||||
assert_recreated_bucket_fallback_has_its_own_owner("queued").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_recreated_bucket_does_not_merge_into_active_old_incarnation() {
|
||||
assert_recreated_bucket_fallback_has_its_own_owner("active").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_recreated_bucket_does_not_merge_into_retrying_old_incarnation() {
|
||||
assert_recreated_bucket_fallback_has_its_own_owner("retrying").await;
|
||||
}
|
||||
|
||||
fn incarnation_bound_mrf_fallback_for_version(incarnation: Uuid, version_id: Option<&str>) -> HealRequest {
|
||||
let mut request = incarnation_bound_mrf_fallback(incarnation);
|
||||
if let HealType::Object { version_id: id, .. } = &mut request.heal_type {
|
||||
*id = version_id.map(str::to_owned);
|
||||
} else {
|
||||
unreachable!("fallback fixture must be an object request");
|
||||
}
|
||||
request
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn versioned_mrf_fallbacks_merge_into_queued_owner() {
|
||||
for version in [Uuid::nil(), Uuid::new_v4()] {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let version_id = version.to_string();
|
||||
let original = incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id));
|
||||
let original_id = original.id.clone();
|
||||
let accepted = manager
|
||||
.submit_heal_request_with_receipt(original)
|
||||
.await
|
||||
.expect("first queued fallback");
|
||||
assert_eq!(accepted.result, HealAdmissionResult::Accepted);
|
||||
let merged = manager
|
||||
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback_for_version(incarnation, Some(&version_id)))
|
||||
.await
|
||||
.expect("duplicate queued fallback");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, original_id);
|
||||
assert_eq!(manager.heal_queue.lock().await.len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_fallback_same_incarnation_merges_but_version_and_scope_remain_distinct() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let original = incarnation_bound_mrf_fallback(incarnation);
|
||||
let original_id = original.id.clone();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(original)
|
||||
.await
|
||||
.expect("first fallback")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let same = manager
|
||||
.submit_heal_request_with_receipt(incarnation_bound_mrf_fallback(incarnation))
|
||||
.await
|
||||
.expect("same-incarnation fallback");
|
||||
assert_eq!(same.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(same.task_id, original_id);
|
||||
|
||||
let mut versioned = incarnation_bound_mrf_fallback(incarnation);
|
||||
versioned.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: "object".to_owned(),
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
};
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(versioned)
|
||||
.await
|
||||
.expect("different version")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let mut other_scope = incarnation_bound_mrf_fallback(incarnation);
|
||||
other_scope.options.pool_index = Some(1);
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(other_scope)
|
||||
.await
|
||||
.expect("different pool")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let mut same_id_other_incarnation = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
same_id_other_incarnation.id = original_id;
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(same_id_other_incarnation)
|
||||
.await
|
||||
.expect("same-ID conflict must be decided")
|
||||
.result,
|
||||
HealAdmissionResult::Dropped(HealAdmissionDropReason::AlreadyRunning)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_bound_dedup_keys_separate_incarnation_source_and_untrusted_components() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let bound = incarnation_bound_mrf_fallback(incarnation);
|
||||
let mut unbound = bound.clone();
|
||||
unbound.expected_mrf_bucket_incarnation_id = None;
|
||||
let mut non_mrf = bound.clone();
|
||||
non_mrf.source = HealRequestSource::ReadRepair;
|
||||
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&unbound));
|
||||
assert_eq!(
|
||||
PriorityHealQueue::make_dedup_key(&non_mrf),
|
||||
PriorityHealQueue::make_dedup_key_for_scope(&non_mrf.heal_type, &non_mrf.options),
|
||||
"an unrelated source retains its previous scope key even if the MRF-only field is set"
|
||||
);
|
||||
assert_eq!(PriorityHealQueue::make_dedup_key(&non_mrf), PriorityHealQueue::make_dedup_key(&unbound));
|
||||
|
||||
let mut left = bound.clone();
|
||||
left.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: "part:a".to_owned(),
|
||||
version_id: Some("b".to_owned()),
|
||||
};
|
||||
let mut right = bound.clone();
|
||||
right.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: "part".to_owned(),
|
||||
version_id: Some("a:b".to_owned()),
|
||||
};
|
||||
assert_eq!(
|
||||
PriorityHealQueue::make_dedup_key_for_scope(&left.heal_type, &left.options),
|
||||
PriorityHealQueue::make_dedup_key_for_scope(&right.heal_type, &right.options),
|
||||
"the legacy delimiter-only key demonstrates the collision"
|
||||
);
|
||||
assert_ne!(
|
||||
PriorityHealQueue::make_dedup_key(&left),
|
||||
PriorityHealQueue::make_dedup_key(&right),
|
||||
"length-prefixed object and version components must not alias"
|
||||
);
|
||||
let mut forged = unbound;
|
||||
forged.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: PriorityHealQueue::make_dedup_key(&bound),
|
||||
version_id: None,
|
||||
};
|
||||
assert_ne!(PriorityHealQueue::make_dedup_key(&bound), PriorityHealQueue::make_dedup_key(&forged));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_bound_optional_versions_do_not_alias_empty_or_forged_components() {
|
||||
let incarnation = Uuid::new_v4();
|
||||
for decode in [false, true] {
|
||||
let make_request = |object: &str, version_id: Option<&str>| {
|
||||
let mut request = incarnation_bound_mrf_fallback(incarnation);
|
||||
let bucket = "mrf-recreated-bucket".to_owned();
|
||||
let object = object.to_owned();
|
||||
let version_id = version_id.map(str::to_owned);
|
||||
request.heal_type = if decode {
|
||||
HealType::ECDecode {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
} else {
|
||||
HealType::Object {
|
||||
bucket,
|
||||
object,
|
||||
version_id,
|
||||
}
|
||||
};
|
||||
request
|
||||
};
|
||||
let absent = make_request("object", None);
|
||||
let empty = make_request("object", Some(""));
|
||||
let forged_object = make_request("object4:some0:", None);
|
||||
let forged_version = make_request("object", Some("4:none"));
|
||||
let keys = [&absent, &empty, &forged_object, &forged_version].map(PriorityHealQueue::make_dedup_key);
|
||||
for (index, key) in keys.iter().enumerate() {
|
||||
assert!(keys.iter().skip(index + 1).all(|other| key != other));
|
||||
}
|
||||
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
assert_eq!(queue.push(absent.clone()), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(empty.clone()), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(make_request("object", None)), QueuePushOutcome::Merged);
|
||||
assert_eq!(queue.push(make_request("object", Some(""))), QueuePushOutcome::Merged);
|
||||
assert_eq!(queue.len(), 2);
|
||||
|
||||
for source in [HealRequestSource::Mrf, HealRequestSource::ReadRepair] {
|
||||
let mut legacy_absent = absent.clone();
|
||||
let mut legacy_empty = empty.clone();
|
||||
legacy_absent.source = source;
|
||||
legacy_empty.source = source;
|
||||
if source == HealRequestSource::Mrf {
|
||||
legacy_absent.expected_mrf_bucket_incarnation_id = None;
|
||||
legacy_empty.expected_mrf_bucket_incarnation_id = None;
|
||||
}
|
||||
assert_eq!(
|
||||
PriorityHealQueue::make_dedup_key(&legacy_absent),
|
||||
PriorityHealQueue::make_dedup_key(&legacy_empty),
|
||||
"unbound and non-MRF work retain the legacy empty-version key"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_bound_queue_releases_each_incarnation_representative_independently() {
|
||||
let mut queue = PriorityHealQueue::new();
|
||||
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
let old_id = old.id.clone();
|
||||
let old_key = PriorityHealQueue::make_dedup_key(&old);
|
||||
let new = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
let new_id = new.id.clone();
|
||||
let new_key = PriorityHealQueue::make_dedup_key(&new);
|
||||
let mut forced_old = old.clone();
|
||||
forced_old.id = Uuid::new_v4().to_string();
|
||||
forced_old.force_start = true;
|
||||
let forced_id = forced_old.id.clone();
|
||||
|
||||
assert_eq!(queue.push(old), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(new), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.push(forced_old), QueuePushOutcome::Accepted);
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(old_id.as_str()));
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
|
||||
queue
|
||||
.remove_request_id(&old_id)
|
||||
.expect("remove the old incarnation representative");
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&old_key), Some(forced_id.as_str()));
|
||||
assert_eq!(queue.queued_request_id_for_dedup_key(&new_key), Some(new_id.as_str()));
|
||||
assert_eq!(queue.pop_next().expect("new incarnation stays queued").id, new_id);
|
||||
assert!(queue.queued_request_id_for_dedup_key(&new_key).is_none());
|
||||
assert_eq!(queue.pop_next().expect("forced old owner stays queued").id, forced_id);
|
||||
assert!(queue.queued_request_id_for_dedup_key(&old_key).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_mrf_recreated_bucket_is_not_merged_into_queued_old_fallback() {
|
||||
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
|
||||
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let old = incarnation_bound_mrf_fallback(Uuid::new_v4());
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(old)
|
||||
.await
|
||||
.expect("old fallback")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let request = incarnation_bound_mrf_fallback(incarnation);
|
||||
let request_id = request.id.clone();
|
||||
let mut intent = MrfIntent {
|
||||
bucket: Arc::from("mrf-recreated-bucket"),
|
||||
object: Arc::from("object"),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
delete_marker_purge: None,
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
}),
|
||||
lease: None,
|
||||
enqueued_at_ms: 1,
|
||||
attempts: 0,
|
||||
};
|
||||
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
|
||||
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
|
||||
let target = MrfRepairNoticeTarget {
|
||||
bucket: Arc::clone(&intent.bucket),
|
||||
object: Arc::clone(&intent.object),
|
||||
version_id: intent.version_id,
|
||||
kind: intent.kind,
|
||||
scope: intent.scope,
|
||||
delete_marker_purge: None,
|
||||
lease: intent.lease,
|
||||
durable_anchor: Some(anchor.clone()),
|
||||
};
|
||||
let receipt = manager
|
||||
.submit_mrf_heal_request_with_receipt_and_identity(request, target)
|
||||
.await
|
||||
.expect("durable admission must be decided");
|
||||
assert_eq!(
|
||||
receipt.result,
|
||||
HealAdmissionResult::Accepted,
|
||||
"a durable new incarnation needs its own owner"
|
||||
);
|
||||
assert_eq!(receipt.task_id, request_id);
|
||||
assert!(
|
||||
manager
|
||||
.heal_queue
|
||||
.lock()
|
||||
.await
|
||||
.requests()
|
||||
.any(|queued| { queued.id == request_id && queued.expected_mrf_bucket_incarnation_id == Some(incarnation) })
|
||||
);
|
||||
assert!(
|
||||
manager
|
||||
.try_in_flight_durable_mrf_anchors()
|
||||
.expect("stable owner snapshot")
|
||||
.contains(&anchor)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_mrf_same_incarnation_merge_retains_the_durable_anchor() {
|
||||
use rustfs_common::mrf_channel::{MrfIngressResult, MrfIntent, MrfKind, MrfScope, try_rearm_mrf_replay_intent};
|
||||
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
let incarnation = Uuid::new_v4();
|
||||
let object = format!("same-generation-{}", Uuid::new_v4());
|
||||
let make_request = || {
|
||||
let mut request = incarnation_bound_mrf_fallback(incarnation);
|
||||
request.heal_type = HealType::Object {
|
||||
bucket: "mrf-recreated-bucket".to_owned(),
|
||||
object: object.clone(),
|
||||
version_id: None,
|
||||
};
|
||||
request
|
||||
};
|
||||
let old = make_request();
|
||||
let old_id = old.id.clone();
|
||||
assert_eq!(
|
||||
manager
|
||||
.submit_heal_request_with_receipt(old)
|
||||
.await
|
||||
.expect("old fallback")
|
||||
.result,
|
||||
HealAdmissionResult::Accepted
|
||||
);
|
||||
let mut intent = MrfIntent {
|
||||
bucket: Arc::from("mrf-recreated-bucket"),
|
||||
object: Arc::from(object.as_str()),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
delete_marker_purge: None,
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
}),
|
||||
lease: None,
|
||||
enqueued_at_ms: 1,
|
||||
attempts: 0,
|
||||
};
|
||||
assert_eq!(try_rearm_mrf_replay_intent(&mut intent), MrfIngressResult::Enqueued);
|
||||
let anchor = rustfs_common::mrf_channel::MrfDurableRepairAnchor::from_intent(&intent, incarnation).expect("durable identity");
|
||||
let target = MrfRepairNoticeTarget {
|
||||
bucket: Arc::clone(&intent.bucket),
|
||||
object: Arc::clone(&intent.object),
|
||||
version_id: intent.version_id,
|
||||
kind: intent.kind,
|
||||
scope: intent.scope,
|
||||
delete_marker_purge: None,
|
||||
lease: intent.lease,
|
||||
durable_anchor: Some(anchor.clone()),
|
||||
};
|
||||
let merged = manager
|
||||
.submit_mrf_heal_request_with_receipt_and_identity(make_request(), target)
|
||||
.await
|
||||
.expect("durable same-incarnation admission");
|
||||
assert_eq!(merged.result, HealAdmissionResult::Merged);
|
||||
assert_eq!(merged.task_id, old_id);
|
||||
assert!(
|
||||
manager
|
||||
.try_in_flight_durable_mrf_anchors()
|
||||
.expect("stable owner snapshot")
|
||||
.contains(&anchor)
|
||||
);
|
||||
}
|
||||
|
||||
fn completed_retention_fixture(completed_at: SystemTime) -> CompletedHealStatus {
|
||||
CompletedHealStatus {
|
||||
task_id: "fixture-token".to_string(),
|
||||
|
||||
+232
-13
@@ -7,8 +7,14 @@ use rustfs_concurrency::{
|
||||
workload::{ForegroundPressure, foreground_pressure},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::HealRequestSource;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio::{sync::Mutex, time::Instant};
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU32, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{sync::Mutex as AsyncMutex, time::Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Default)]
|
||||
@@ -22,17 +28,19 @@ pub(crate) enum PacingDecision {
|
||||
Wait(Option<ForegroundPressure>),
|
||||
}
|
||||
|
||||
/// Cooperative pacing for one heal execution, not a storage admission permit.
|
||||
pub(crate) struct MainlinePacer {
|
||||
/// Foreground watermarks for one heal execution. Constructed once per request;
|
||||
/// disabled when watermarks are zero or no workload provider is configured.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ForegroundPacing {
|
||||
provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
|
||||
read_high: usize,
|
||||
write_high: usize,
|
||||
pause: Duration,
|
||||
source: HealRequestSource,
|
||||
state: Mutex<PacingState>,
|
||||
state: Arc<AsyncMutex<PacingState>>,
|
||||
}
|
||||
|
||||
impl MainlinePacer {
|
||||
impl ForegroundPacing {
|
||||
pub(crate) fn new(
|
||||
provider: Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>,
|
||||
read_high: usize,
|
||||
@@ -49,7 +57,7 @@ impl MainlinePacer {
|
||||
write_high: write_high.min(100),
|
||||
pause: pause.min(Duration::from_secs(1)),
|
||||
source,
|
||||
state: Mutex::new(PacingState::default()),
|
||||
state: Arc::new(AsyncMutex::new(PacingState::default())),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -125,6 +133,142 @@ impl MainlinePacer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Single owner of "how hard may this heal pass push storage?".
|
||||
///
|
||||
/// One instance spans the whole erasure-set scan (all buckets and pages):
|
||||
/// - foreground admission pressure (`ForegroundPacing`)
|
||||
/// - S3 `SlowDown` ("Please reduce your request rate") self-backpressure
|
||||
///
|
||||
/// Call `observe_object_outcome` exactly once per finished object so a single
|
||||
/// SlowDown cannot double-reduce concurrency.
|
||||
pub(crate) struct HealPageRate {
|
||||
base_concurrency: usize,
|
||||
/// 0 = full base concurrency; each SlowDown increases the reduction.
|
||||
slow_down_step: AtomicU32,
|
||||
slow_down_until: std::sync::Mutex<Option<Instant>>,
|
||||
foreground: Option<ForegroundPacing>,
|
||||
}
|
||||
|
||||
impl HealPageRate {
|
||||
/// Always-on controller. Foreground yield is optional; SlowDown control is not.
|
||||
pub(crate) fn new(base_concurrency: usize, foreground: Option<ForegroundPacing>) -> Self {
|
||||
Self {
|
||||
base_concurrency: base_concurrency.max(1),
|
||||
slow_down_step: AtomicU32::new(0),
|
||||
slow_down_until: std::sync::Mutex::new(None),
|
||||
foreground,
|
||||
}
|
||||
}
|
||||
|
||||
/// Concurrency after applying the current SlowDown reduction. Always ≥ 1
|
||||
/// so maintenance can still make progress under sustained rate pressure.
|
||||
pub(crate) fn effective_concurrency(&self) -> usize {
|
||||
let step = self.slow_down_step.load(Ordering::Relaxed).min(8);
|
||||
(self.base_concurrency >> step).max(1)
|
||||
}
|
||||
|
||||
/// Fold one finished object into the rate state. Call once per outcome.
|
||||
/// `saw_slow_down` is true when any attempt of this object hit SlowDown,
|
||||
/// including attempts that later succeeded on retry.
|
||||
pub(crate) fn observe_object_outcome(&self, result: &std::result::Result<bool, &Error>, saw_slow_down: bool) {
|
||||
if saw_slow_down || matches!(result, Err(err) if err.is_slow_down()) {
|
||||
self.observe_slow_down();
|
||||
return;
|
||||
}
|
||||
if result.is_ok() {
|
||||
self.observe_success();
|
||||
}
|
||||
}
|
||||
|
||||
/// S3 `SlowDown` is a rate signal: cut fan-out and arm exponential cool-down.
|
||||
fn observe_slow_down(&self) {
|
||||
// fetch_add returns the previous step; +1 so the first SlowDown uses
|
||||
// exponent 1 (2s), not exponent 0 (1s).
|
||||
let step = self.slow_down_step.fetch_add(1, Ordering::Relaxed).saturating_add(1).min(8);
|
||||
let delay = Self::backoff(step);
|
||||
let ready = Instant::now() + delay;
|
||||
if let Ok(mut until) = self.slow_down_until.lock()
|
||||
&& until.is_none_or(|current| current < ready)
|
||||
{
|
||||
*until = Some(ready);
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore one concurrency step only after the SlowDown cool-down expires.
|
||||
/// A same-page success must not undo the reduction while the pass is still
|
||||
/// in the rate-limited window (the page semaphore is fixed until the next page).
|
||||
fn observe_success(&self) {
|
||||
let cooling = self
|
||||
.slow_down_until
|
||||
.lock()
|
||||
.ok()
|
||||
.and_then(|until| *until)
|
||||
.is_some_and(|ready| ready > Instant::now());
|
||||
if cooling {
|
||||
return;
|
||||
}
|
||||
self.slow_down_step
|
||||
.try_update(Ordering::Relaxed, Ordering::Relaxed, |step| Some(step.saturating_sub(1)))
|
||||
.ok();
|
||||
}
|
||||
|
||||
/// Wait out SlowDown cool-down, then the foreground pacer. Cancellation interrupts.
|
||||
pub(crate) async fn wait_before_page(&self, cancel: &CancellationToken) -> Result<()> {
|
||||
self.wait_slow_down(cancel).await?;
|
||||
if let Some(foreground) = &self.foreground {
|
||||
foreground.wait(cancel).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Foreground admission decision while the caller owns page capacity.
|
||||
pub(crate) fn admission_decision(&self) -> PacingDecision {
|
||||
match &self.foreground {
|
||||
Some(foreground) => foreground.admission_decision(),
|
||||
None => PacingDecision::Ready,
|
||||
}
|
||||
}
|
||||
|
||||
/// Foreground wait after a rejected admission. Returns whether a pause was paid.
|
||||
pub(crate) async fn wait_after_admission(
|
||||
&self,
|
||||
cancel: &CancellationToken,
|
||||
observed: Option<ForegroundPressure>,
|
||||
) -> Result<bool> {
|
||||
match &self.foreground {
|
||||
Some(foreground) => foreground.wait_after_admission(cancel, observed).await,
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_slow_down(&self, cancel: &CancellationToken) -> Result<()> {
|
||||
let delay = self
|
||||
.slow_down_until
|
||||
.lock()
|
||||
.ok()
|
||||
.and_then(|until| *until)
|
||||
.map(|ready| ready.saturating_duration_since(Instant::now()));
|
||||
let Some(delay) = delay else {
|
||||
return Ok(());
|
||||
};
|
||||
if delay.is_zero() {
|
||||
return Ok(());
|
||||
}
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = cancel.cancelled() => Err(Error::TaskCancelled),
|
||||
_ = tokio::time::sleep(delay) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn backoff(step: u32) -> Duration {
|
||||
// step 1 => 2s, 2 => 4s, … capped at 30s — S3 clients are told to reduce
|
||||
// rate with exponential backoff, and heal must do the same to itself.
|
||||
let exponent = step.clamp(1, 5);
|
||||
Duration::from_secs(2_u64.saturating_pow(exponent)).min(Duration::from_secs(30))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct TestPressure {
|
||||
pub(crate) active: std::sync::atomic::AtomicUsize,
|
||||
@@ -164,12 +308,87 @@ mod tests {
|
||||
use rustfs_concurrency::WorkloadClass;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_first_slow_down_waits_two_seconds() {
|
||||
let rate = HealPageRate::new(8, None);
|
||||
rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
|
||||
let cancel = CancellationToken::new();
|
||||
let before = Instant::now();
|
||||
rate.wait_before_page(&cancel).await.expect("cool-down wait");
|
||||
assert_eq!(Instant::now() - before, Duration::from_secs(2));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_reduces_concurrency_and_backs_off() {
|
||||
let rate = HealPageRate::new(8, None);
|
||||
assert_eq!(rate.effective_concurrency(), 8);
|
||||
rate.observe_object_outcome(&Err(&Error::transient_skip("Please reduce your request rate")), false);
|
||||
assert_eq!(rate.effective_concurrency(), 4);
|
||||
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
|
||||
assert_eq!(rate.effective_concurrency(), 2);
|
||||
// Floor is one so maintenance never stops entirely.
|
||||
for _ in 0..8 {
|
||||
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), true);
|
||||
}
|
||||
assert_eq!(rate.effective_concurrency(), 1);
|
||||
|
||||
let cancel = CancellationToken::new();
|
||||
assert!(rate.wait_before_page(&cancel).await.is_ok());
|
||||
|
||||
// After cool-down, successes restore concurrency one step at a time.
|
||||
for _ in 0..12 {
|
||||
rate.observe_object_outcome(&Ok(true), false);
|
||||
}
|
||||
assert!(rate.effective_concurrency() > 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_same_page_success_does_not_undo_slow_down() {
|
||||
let rate = HealPageRate::new(8, None);
|
||||
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
|
||||
assert_eq!(rate.effective_concurrency(), 4);
|
||||
// Same-page successes while cool-down is armed must not restore fan-out.
|
||||
for _ in 0..4 {
|
||||
rate.observe_object_outcome(&Ok(true), false);
|
||||
}
|
||||
assert_eq!(rate.effective_concurrency(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_slow_down_then_retry_success_still_reduces() {
|
||||
let rate = HealPageRate::new(8, None);
|
||||
// Replacement retry: SlowDown on attempt 1, success on attempt 2.
|
||||
rate.observe_object_outcome(&Ok(true), true);
|
||||
assert_eq!(rate.effective_concurrency(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_wait_is_cancellable() {
|
||||
let rate = HealPageRate::new(4, None);
|
||||
rate.observe_object_outcome(&Err(&Error::Storage(crate::heal::EcstoreError::SlowDown)), false);
|
||||
let cancel = CancellationToken::new();
|
||||
cancel.cancel();
|
||||
assert!(matches!(rate.wait_before_page(&cancel).await, Err(Error::TaskCancelled)));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_page_rate_still_waits_for_foreground_after_slow_down_clears() {
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundWrite, 100));
|
||||
let foreground = ForegroundPacing::new(provider, 0, 80, Duration::from_millis(250), HealRequestSource::AutoHeal)
|
||||
.expect("foreground pacer");
|
||||
let rate = HealPageRate::new(4, Some(foreground));
|
||||
let cancel = CancellationToken::new();
|
||||
let before = Instant::now();
|
||||
rate.wait_before_page(&cancel).await.expect("bounded foreground pause");
|
||||
assert_eq!(Instant::now() - before, Duration::from_millis(250));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_hysteresis_uses_configured_watermarks_and_stable_low_window() {
|
||||
for class in [WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite] {
|
||||
let provider = Arc::new(TestPressure::new(class, 0));
|
||||
let pause = Duration::from_millis(250);
|
||||
let pacer = MainlinePacer::new(
|
||||
let pacer = ForegroundPacing::new(
|
||||
provider.clone(),
|
||||
if class == WorkloadClass::ForegroundRead { 40 } else { 0 },
|
||||
if class == WorkloadClass::ForegroundWrite { 40 } else { 0 },
|
||||
@@ -197,9 +416,9 @@ mod tests {
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn running_mainline_huge_pause_is_capped_and_disabled_classes_do_not_sleep() {
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
|
||||
assert!(MainlinePacer::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
|
||||
assert!(MainlinePacer::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
|
||||
let pacer = MainlinePacer::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
|
||||
assert!(ForegroundPacing::new(provider.clone(), 0, 0, Duration::from_secs(1), HealRequestSource::Admin).is_none());
|
||||
assert!(ForegroundPacing::new(provider.clone(), 80, 80, Duration::ZERO, HealRequestSource::Admin).is_none());
|
||||
let pacer = ForegroundPacing::new(provider, 80, 80, Duration::from_secs(3600), HealRequestSource::Admin).expect("pacer");
|
||||
let before = Instant::now();
|
||||
pacer.wait(&CancellationToken::new()).await.expect("hard-capped pause");
|
||||
assert_eq!(Instant::now() - before, Duration::from_secs(1));
|
||||
@@ -209,7 +428,7 @@ mod tests {
|
||||
async fn running_mainline_waiters_cancel_and_task_latches_are_isolated() {
|
||||
let provider = Arc::new(TestPressure::new(WorkloadClass::ForegroundRead, 100));
|
||||
let paced = Arc::new(
|
||||
MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
|
||||
ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin).expect("pacer"),
|
||||
);
|
||||
let cancel_first = CancellationToken::new();
|
||||
let first = tokio::spawn({
|
||||
@@ -228,7 +447,7 @@ mod tests {
|
||||
cancel_second.cancel();
|
||||
assert!(matches!(second.await.expect("queued waiter"), Err(Error::TaskCancelled)));
|
||||
provider.active.store(0, Ordering::SeqCst);
|
||||
let other_task = MainlinePacer::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
|
||||
let other_task = ForegroundPacing::new(provider.clone(), 80, 80, Duration::from_secs(1), HealRequestSource::Admin)
|
||||
.expect("independent task");
|
||||
let before = Instant::now();
|
||||
other_task
|
||||
|
||||
@@ -35,7 +35,9 @@ mod replacement;
|
||||
mod utils;
|
||||
|
||||
pub(crate) use checkpoint::AdminErasureCheckpoint;
|
||||
pub use checkpoint::{CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, ResumeCheckpoint};
|
||||
pub use checkpoint::{
|
||||
CheckpointManager, CheckpointObjectOutcome, CheckpointObjectOutcomeRecord, CheckpointProgressSnapshot, ResumeCheckpoint,
|
||||
};
|
||||
pub(crate) use gc::ResumeGc;
|
||||
pub use handoff::{ReplacementHandoff, ReplacementHandoffLink, ReplacementHandoffPhase};
|
||||
pub use legacy_handoff::{LegacyReplacementApproval, LegacyReplacementSource};
|
||||
|
||||
@@ -152,6 +152,42 @@ impl AdminErasureCheckpoint {
|
||||
}
|
||||
}
|
||||
|
||||
/// Lightweight progress view for resume decisions. Omits the per-page object
|
||||
/// ledgers so callers do not clone a million-object checkpoint just to read
|
||||
/// counters and positions.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct CheckpointProgressSnapshot {
|
||||
pub current_bucket_index: usize,
|
||||
pub current_object_index: usize,
|
||||
pub successful_objects: u64,
|
||||
pub failed_object_count: u64,
|
||||
pub skipped_object_count: u64,
|
||||
pub skipped_new_versions: u64,
|
||||
pub skipped_ilm_expired: u64,
|
||||
pub processed_bytes: u64,
|
||||
pub total_objects: u64,
|
||||
pub total_bytes: u64,
|
||||
pub baseline_generation: Option<u64>,
|
||||
pub baseline_known: bool,
|
||||
pub counter_unknown: bool,
|
||||
}
|
||||
|
||||
impl CheckpointProgressSnapshot {
|
||||
pub fn has_progress(&self) -> bool {
|
||||
self.baseline_known
|
||||
|| self.successful_objects > 0
|
||||
|| self.failed_object_count > 0
|
||||
|| self.skipped_object_count > 0
|
||||
|| self.skipped_new_versions > 0
|
||||
|| self.skipped_ilm_expired > 0
|
||||
|| self.processed_bytes > 0
|
||||
|| self.total_objects > 0
|
||||
|| self.total_bytes > 0
|
||||
|| self.baseline_generation.is_some()
|
||||
|| self.counter_unknown
|
||||
}
|
||||
}
|
||||
|
||||
/// resume checkpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResumeCheckpoint {
|
||||
@@ -715,6 +751,75 @@ impl CheckpointManager {
|
||||
self.checkpoint.read().await.clone()
|
||||
}
|
||||
|
||||
/// True when a page object is already settled in the current ledger.
|
||||
/// Membership is checked under the lock without cloning the checkpoint,
|
||||
/// which used to copy every page's dedup sets on the small-object path.
|
||||
pub async fn is_object_settled(&self, key: &str) -> bool {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
checkpoint.processed_objects.contains(key)
|
||||
|| checkpoint.failed_objects.contains(key)
|
||||
|| checkpoint.skipped_objects.contains(key)
|
||||
}
|
||||
|
||||
/// True when the administrator checkpoint already finished.
|
||||
pub async fn admin_completed(&self) -> bool {
|
||||
self.checkpoint
|
||||
.read()
|
||||
.await
|
||||
.admin
|
||||
.as_ref()
|
||||
.is_some_and(|admin| admin.completed)
|
||||
}
|
||||
|
||||
/// Current resume position without cloning the object ledgers.
|
||||
pub async fn resume_position(&self) -> (usize, usize) {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
(checkpoint.current_bucket_index, checkpoint.current_object_index)
|
||||
}
|
||||
|
||||
/// Dangling-delete grace summary without cloning the object ledgers.
|
||||
pub async fn dangling_delete_grace(&self) -> (u64, Option<u64>, bool) {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
(
|
||||
checkpoint.dangling_delete_grace_objects,
|
||||
checkpoint.dangling_delete_retry_not_before,
|
||||
checkpoint.counter_unknown,
|
||||
)
|
||||
}
|
||||
|
||||
/// Aggregate progress counters without cloning the page dedup sets.
|
||||
pub async fn progress_snapshot(&self) -> CheckpointProgressSnapshot {
|
||||
let checkpoint = self.checkpoint.read().await;
|
||||
CheckpointProgressSnapshot {
|
||||
current_bucket_index: checkpoint.current_bucket_index,
|
||||
current_object_index: checkpoint.current_object_index,
|
||||
successful_objects: checkpoint.successful_objects,
|
||||
failed_object_count: checkpoint.failed_object_count,
|
||||
skipped_object_count: checkpoint.skipped_object_count,
|
||||
skipped_new_versions: checkpoint.skipped_new_versions,
|
||||
skipped_ilm_expired: checkpoint.skipped_ilm_expired,
|
||||
processed_bytes: checkpoint.processed_bytes,
|
||||
total_objects: checkpoint.total_objects,
|
||||
total_bytes: checkpoint.total_bytes,
|
||||
baseline_generation: checkpoint.baseline_generation,
|
||||
baseline_known: checkpoint.baseline_known,
|
||||
counter_unknown: checkpoint.counter_unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/// Finish the administrator outcome in place and persist it.
|
||||
pub async fn complete_admin(&self) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
let admin = checkpoint
|
||||
.admin
|
||||
.as_mut()
|
||||
.ok_or_else(|| Error::InvalidCheckpoint("Missing administrator outcome at completion".to_string()))?;
|
||||
admin.completed = true;
|
||||
admin.outcome.finish(None);
|
||||
drop(checkpoint);
|
||||
self.save_checkpoint().await
|
||||
}
|
||||
|
||||
/// update position
|
||||
pub async fn update_position(&self, bucket_index: usize, object_index: usize) -> Result<()> {
|
||||
let mut checkpoint = self.checkpoint.write().await;
|
||||
|
||||
@@ -485,7 +485,7 @@ pub struct HealTask {
|
||||
pub cancel_token: tokio_util::sync::CancellationToken,
|
||||
/// Storage layer interface
|
||||
pub storage: Arc<dyn HealStorageAPI>,
|
||||
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
||||
mainline_pacer: Option<super::pacing::ForegroundPacing>,
|
||||
}
|
||||
|
||||
impl HealTask {
|
||||
@@ -586,7 +586,7 @@ impl HealTask {
|
||||
task
|
||||
}
|
||||
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<Arc<super::pacing::MainlinePacer>>) -> Self {
|
||||
pub(crate) fn with_mainline_pacer(mut self, pacer: Option<super::pacing::ForegroundPacing>) -> Self {
|
||||
self.mainline_pacer = pacer;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -21,9 +21,6 @@ const MAX_DEFERRED_OBJECTS: usize = 256;
|
||||
const MAX_DEFERRED_BYTES: usize = 256 * 1024;
|
||||
const MAX_DEFERRED_FORWARD_PAGES: u64 = 2;
|
||||
const MAX_DEFERRED_AGE: Duration = Duration::from_secs(30);
|
||||
// A recovered pool can become writable just after the ordinary retry budget.
|
||||
// Keep this extra attempt inside the existing deferred age and backoff caps.
|
||||
const MAX_SLOWDOWN_OBJECT_RETRIES: u32 = MAX_BUCKET_OBJECT_HEAL_RETRIES + 1;
|
||||
|
||||
struct DeferredObject {
|
||||
name: String,
|
||||
@@ -915,12 +912,7 @@ impl HealTask {
|
||||
);
|
||||
} else if !age_exhausted
|
||||
&& err.is_recoverable_object_heal()
|
||||
&& retry_attempt
|
||||
< if matches!(&err, Error::Storage(crate::heal::StorageError::SlowDown)) {
|
||||
MAX_SLOWDOWN_OBJECT_RETRIES
|
||||
} else {
|
||||
MAX_BUCKET_OBJECT_HEAL_RETRIES
|
||||
}
|
||||
&& retry_attempt < MAX_BUCKET_OBJECT_HEAL_RETRIES
|
||||
{
|
||||
terminal_outcome = false;
|
||||
debug!(
|
||||
|
||||
@@ -480,8 +480,9 @@ impl HealTask {
|
||||
message: format!("Missing source bucket incarnation for durable MRF repair {bucket}/{object}"),
|
||||
})?;
|
||||
if current_bucket_incarnation_id != bucket_incarnation_id {
|
||||
return Err(Error::TaskExecutionFailed {
|
||||
message: format!("Bucket incarnation changed before durable MRF repair {bucket}/{object}"),
|
||||
return Err(Error::StaleBucketIncarnation {
|
||||
bucket: bucket.to_owned(),
|
||||
expected: Some(bucket_incarnation_id),
|
||||
});
|
||||
}
|
||||
expected.bucket_incarnation_id = Some(bucket_incarnation_id);
|
||||
|
||||
@@ -3375,10 +3375,67 @@ async fn durable_mrf_heal_rejects_a_recreated_bucket_before_storage_heal() {
|
||||
|
||||
let result = task.heal_object("bucket", "object", None).await;
|
||||
|
||||
assert!(matches!(result, Err(Error::TaskExecutionFailed { .. })));
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::StaleBucketIncarnation { expected: Some(id), .. }) if id == original_incarnation
|
||||
));
|
||||
assert!(storage.heal_object_calls.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn durable_mrf_stale_incarnation_object_name_cannot_change_retry_classification() {
|
||||
for object in ["timed out", "network error", "ordinary"] {
|
||||
let original_incarnation = Uuid::new_v4();
|
||||
let recreated_incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(recreated_incarnation)),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::object("bucket".to_owned(), object.to_owned(), None);
|
||||
request.source = HealRequestSource::Mrf;
|
||||
request.expected_mrf_bucket_incarnation_id = Some(original_incarnation);
|
||||
request.options.pool_index = Some(0);
|
||||
request.options.set_index = Some(1);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("a recreated bucket must reject the old MRF responsibility");
|
||||
assert!(
|
||||
!error.is_recoverable_heal(),
|
||||
"user-controlled object names must not make a stale incarnation retryable: {object}"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
&error,
|
||||
Error::StaleBucketIncarnation { bucket, expected }
|
||||
if bucket == "bucket" && *expected == Some(original_incarnation)
|
||||
),
|
||||
"the failure must retain the original bucket incarnation: {error:?}"
|
||||
);
|
||||
assert!(
|
||||
matches!(crate::error::classify_heal_error(&error), crate::error::HealErrorClass::Aborted),
|
||||
"stale responsibility must be classified independently of the object name"
|
||||
);
|
||||
assert!(
|
||||
storage.heal_object_calls.lock().expect("object heal calls").is_empty(),
|
||||
"an old responsibility must not reach storage repair in the recreated bucket"
|
||||
);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0, "a stale task cannot claim a repair");
|
||||
assert!(
|
||||
outcome.objects.iter().all(|item| !matches!(
|
||||
&item.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
)),
|
||||
"the stale task must not produce a positive repair proof"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn read_repair_object_heal_is_not_failed_by_flat_task_timeout() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
|
||||
@@ -220,29 +220,6 @@ async fn full_window_abort_accounts_inline_once_and_leaves_unstarted_tail_unproc
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn slowdown_can_recover_within_existing_deferred_age_budget() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"]]));
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert("a".to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
|
||||
let task = bucket_task(storage.clone());
|
||||
let started = tokio::time::Instant::now();
|
||||
task.execute()
|
||||
.await
|
||||
.expect("transient slowdown recovers within the deferred age budget");
|
||||
assert!(started.elapsed() < Duration::from_secs(30));
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 5);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(
|
||||
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
|
||||
(1, 0, 4)
|
||||
);
|
||||
assert_eq!(outcome.objects.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
|
||||
let storage = Arc::new(pages_storage(&[&["a"], &["b"], &["c"], &["d"]]));
|
||||
@@ -251,7 +228,7 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("outcomes")
|
||||
.insert(name.to_string(), (0..5).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
|
||||
.insert(name.to_string(), (0..4).map(|_| MockHealObjectOutcome::RetryableSlowDown).collect());
|
||||
}
|
||||
let task = bucket_task(storage.clone());
|
||||
let execution = task.execute();
|
||||
@@ -262,9 +239,9 @@ async fn repeated_slowdown_keeps_attempts_and_forward_pages_bounded() {
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(
|
||||
(outcome.counters.processed, outcome.counters.failed, outcome.counters.attempt_failures),
|
||||
(4, 4, 20)
|
||||
(4, 4, 16)
|
||||
);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 20);
|
||||
assert_eq!(storage.heal_object_calls.lock().expect("calls").len(), 16);
|
||||
for name in ["a", "b", "c", "d"] {
|
||||
assert_eq!(outcome.objects.iter().filter(|item| item.identity.object == name).count(), 1);
|
||||
}
|
||||
|
||||
@@ -16,9 +16,10 @@ use super::{
|
||||
transport::{DiscoveryRequest, OidcHttpError, ReqwestHttpClient, oidc_extra_root_certs, oidc_http_error_diagnostics},
|
||||
};
|
||||
use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
|
||||
use openidconnect::core::{CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
|
||||
use openidconnect::core::{CoreIdToken, CoreIdTokenClaims, CoreIdTokenVerifier, CoreJsonWebKeySet, CoreJwsSigningAlgorithm};
|
||||
use openidconnect::{
|
||||
AsyncHttpClient, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, ProviderMetadataWithLogout, TokenUrl,
|
||||
AsyncHttpClient, ClaimsVerificationError, ClientId, ClientSecret, DiscoveryError, IssuerUrl, JsonWebKeySetUrl, NonceVerifier,
|
||||
ProviderMetadataWithLogout, SignatureVerificationError, TokenUrl,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
@@ -212,6 +213,35 @@ impl DiscoveredProviderMetadata {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn verify_id_token_claims<'a, N: NonceVerifier>(
|
||||
token: &'a CoreIdToken,
|
||||
verifier: &CoreIdTokenVerifier<'_>,
|
||||
nonce_verifier: N,
|
||||
) -> Result<&'a CoreIdTokenClaims, ClaimsVerificationError> {
|
||||
let expected_length = match token.signing_alg().map_err(ClaimsVerificationError::SignatureVerification)? {
|
||||
CoreJwsSigningAlgorithm::HmacSha256 => 32,
|
||||
CoreJwsSigningAlgorithm::HmacSha384 => 48,
|
||||
CoreJwsSigningAlgorithm::HmacSha512 => 64,
|
||||
_ => return token.claims(verifier, nonce_verifier),
|
||||
};
|
||||
// RUSTFS_COMPAT_TODO(openidconnect-hmac-signature-width): Reject widths that upstream converts to fixed-size arrays. Remove after upstream returns verification errors for arbitrary HMAC signature lengths.
|
||||
// openidconnect 4.1.0 converts HMAC signatures to fixed-size arrays before verifying them.
|
||||
let invalid_length = || {
|
||||
ClaimsVerificationError::SignatureVerification(SignatureVerificationError::CryptoError(
|
||||
"invalid HMAC signature length".to_owned(),
|
||||
))
|
||||
};
|
||||
let compact = token.to_string();
|
||||
let (_, signature) = compact.rsplit_once('.').ok_or_else(invalid_length)?;
|
||||
let signature = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(signature)
|
||||
.map_err(|_| invalid_length())?;
|
||||
if signature.len() != expected_length {
|
||||
return Err(invalid_length());
|
||||
}
|
||||
token.claims(verifier, nonce_verifier)
|
||||
}
|
||||
|
||||
// This adapter is used only for discovery/JWKS fetches, never token exchange.
|
||||
struct JwksAcceptClient<'a> {
|
||||
inner: &'a ReqwestHttpClient,
|
||||
|
||||
@@ -242,7 +242,7 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
|
||||
let now = time::OffsetDateTime::now_utc().unix_timestamp();
|
||||
let payload = serde_json::json!({
|
||||
"iss": config.config_url, "sub": "synthetic-subject", "aud": config.client_id,
|
||||
"iat": now, "exp": now + 300,
|
||||
"iat": now, "exp": now + 300, "nonce": "synthetic-nonce",
|
||||
});
|
||||
|
||||
for (algorithm, name, signature_length) in [
|
||||
@@ -275,22 +275,28 @@ fn configured_verifier_rejects_malformed_hmac_signature_lengths() {
|
||||
.expect("parse configured provider metadata");
|
||||
let verifier = metadata.verifier(&config);
|
||||
assert!(
|
||||
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
"valid {name} token must pass for workload={workload}"
|
||||
);
|
||||
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("synthetic-nonce".into())).is_ok());
|
||||
assert!(verify_id_token_claims(&valid, &verifier, &Nonce::new("wrong-nonce".into())).is_err());
|
||||
|
||||
for length in [1, 0, signature_length - 1, signature_length, signature_length + 1, 128] {
|
||||
let signature = base64_simd::URL_SAFE_NO_PAD.encode_to_string(vec![0_u8; length]);
|
||||
let malformed: CoreIdToken = format!("{signing_input}.{signature}")
|
||||
.parse()
|
||||
.expect("invalid signature length must still reach cryptographic verification");
|
||||
.expect("invalid signature length must remain parseable before verification");
|
||||
assert!(
|
||||
malformed.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
verify_id_token_claims(&malformed, &verifier, |_: Option<&Nonce>| Ok(())).is_err(),
|
||||
"{name} signature length {length} must return an error for workload={workload}"
|
||||
);
|
||||
assert!(
|
||||
verify_id_token_claims(&malformed, &verifier, &Nonce::new("synthetic-nonce".into())).is_err(),
|
||||
"{name} signature length {length} must return an error with a nonce for workload={workload}"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
valid.claims(&verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
verify_id_token_claims(&valid, &verifier, |_: Option<&Nonce>| Ok(())).is_ok(),
|
||||
"invalid inputs must not prevent later valid {name} verification"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::{
|
||||
OidcConfigQuery, OidcConfigSnapshot, OidcProviderConfig, OidcProviderSummary, OidcSiteReplicationSnapshot,
|
||||
SourcedOidcProviderConfig, load_oidc_config_snapshot,
|
||||
},
|
||||
provider::{ProviderRuntime, discover_provider_for_recovery},
|
||||
provider::{ProviderRuntime, discover_provider_for_recovery, verify_id_token_claims},
|
||||
state::{OidcAuthSession, OidcLogoutSession, OidcStateStore},
|
||||
transport::{OidcExtraRootCaProvider, ReqwestHttpClient, inspect_token_response_body, oidc_http_error_diagnostics},
|
||||
};
|
||||
@@ -535,7 +535,7 @@ impl OidcSys {
|
||||
let verifier = client
|
||||
.id_token_verifier()
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
let verified = id_token.claims(&verifier, &Nonce::new(session.nonce.clone()));
|
||||
let verified = verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()));
|
||||
if let Err(e) = verified {
|
||||
let verification_error = e.to_string();
|
||||
warn!(
|
||||
@@ -583,8 +583,7 @@ impl OidcSys {
|
||||
let verifier = client
|
||||
.id_token_verifier()
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
id_token
|
||||
.claims(&verifier, &Nonce::new(session.nonce.clone()))
|
||||
verify_id_token_claims(id_token, &verifier, &Nonce::new(session.nonce.clone()))
|
||||
.map_err(|retry_err| {
|
||||
error!(
|
||||
event = EVENT_OIDC_DIAGNOSTICS,
|
||||
@@ -767,7 +766,7 @@ impl OidcSys {
|
||||
.metadata
|
||||
.verifier(&config)
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
if let Err(e) = id_token.claims(&verifier, |_: Option<&Nonce>| Ok(())) {
|
||||
if let Err(e) = verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(())) {
|
||||
state = self
|
||||
.provider_runtime
|
||||
.refresh_provider_state(&provider_id, &config)
|
||||
@@ -780,8 +779,7 @@ impl OidcSys {
|
||||
.metadata
|
||||
.verifier(&config)
|
||||
.set_other_audience_verifier_fn(|aud| trusted_aud(&config.other_audiences, aud));
|
||||
id_token
|
||||
.claims(&verifier, |_: Option<&Nonce>| Ok(()))
|
||||
verify_id_token_claims(&id_token, &verifier, |_: Option<&Nonce>| Ok(()))
|
||||
.map_err(|retry_err| format!("ID token verification failed after JWKS refresh: {retry_err}"))?;
|
||||
}
|
||||
|
||||
|
||||
@@ -1411,6 +1411,28 @@ pub fn record_peer_reachable(addr: &str) {
|
||||
publish_offline_gauge(&peers);
|
||||
}
|
||||
|
||||
/// Record a successful interaction with a cluster peer, but only when the peer is currently
|
||||
/// marked offline. This is the fast recovery path for cached-channel RPCs, recovery probes,
|
||||
/// and lock-path successes that do not go through a fresh dial.
|
||||
///
|
||||
/// Fast path (peer unknown or already online): a shared read lock, no write, no gauge
|
||||
/// republish — the overwhelmingly common case on a healthy cluster. Slow path (offline →
|
||||
/// online transition): upgrades to the write lock and delegates to [`record_peer_reachable`].
|
||||
pub fn record_peer_reachable_if_offline(addr: &str) {
|
||||
let key = normalize_peer_key(addr);
|
||||
// Fast path: read lock to check whether the peer is currently offline.
|
||||
{
|
||||
let peers = CLUSTER_PEER_HEALTH.read().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
match peers.get(key) {
|
||||
None => return,
|
||||
Some(entry) if entry.online => return,
|
||||
Some(_) => {} // offline: fall through to the write path
|
||||
}
|
||||
}
|
||||
// Slow path: the peer is offline, so flip it back online and republish the gauge.
|
||||
record_peer_reachable(addr);
|
||||
}
|
||||
|
||||
/// Record a failed interaction with a cluster peer (dial failure or RPC-triggered eviction). After
|
||||
/// `failure_threshold` (>= 1) consecutive failures the peer flips offline.
|
||||
pub fn record_peer_unreachable(addr: &str, failure_threshold: u32) {
|
||||
@@ -2188,6 +2210,34 @@ mod tests {
|
||||
assert_eq!(CLUSTER_SERVERS_OFFLINE_TOTAL, "rustfs_cluster_servers_offline_total");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_peer_reachable_if_offline_recovers_stale_offline_state() {
|
||||
let addr = "http://cluster-peer-recover-if-offline-test:9000";
|
||||
|
||||
// Mark the peer offline.
|
||||
record_peer_unreachable(addr, 1);
|
||||
assert!(cluster_peer_is_offline(addr));
|
||||
|
||||
// A successful cached-channel RPC (no fresh dial) clears the offline state.
|
||||
record_peer_reachable_if_offline(addr);
|
||||
assert!(!cluster_peer_is_offline(addr));
|
||||
assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_peer_reachable_if_offline_no_op_for_online_peer() {
|
||||
let addr = "http://cluster-peer-recover-noop-test:9000";
|
||||
|
||||
record_peer_reachable(addr);
|
||||
// Should not panic or change state for an already-online peer.
|
||||
record_peer_reachable_if_offline(addr);
|
||||
assert_eq!(cluster_peer_observed_online_status(addr), Some(true));
|
||||
|
||||
// Should also be a no-op for unknown peers.
|
||||
record_peer_reachable_if_offline("http://cluster-peer-recover-unknown-test:9000");
|
||||
assert_eq!(cluster_peer_observed_online_status("http://cluster-peer-recover-unknown-test:9000"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_peer_should_bypass_is_self_healing() {
|
||||
let addr = "http://cluster-peer-bypass-selfheal-test:9000";
|
||||
|
||||
@@ -382,42 +382,6 @@ impl LockClient for LocalClient {
|
||||
}
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let mut guards = shard.write().await;
|
||||
let Some(entry) = guards.get_mut(lock_id) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if entry.is_expired() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(entry.guard.downgrade_committed_tail())
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let mut guards = shard.write().await;
|
||||
let Some(entry) = guards.get_mut(lock_id) else { return Ok(false) };
|
||||
if entry.is_expired() {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(entry.guard.try_upgrade_committed_tail())
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
self.refresh(lock_id).await
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let shard = self.get_shard(lock_id);
|
||||
let guards = shard.read().await;
|
||||
let Some(entry) = guards.get(lock_id) else { return Ok(None) };
|
||||
if entry.is_expired() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(entry.guard.waiting_writer())
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
self.release(lock_id).await
|
||||
}
|
||||
@@ -583,32 +547,6 @@ mod tests {
|
||||
assert!(!response.success);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn downgrade_tail_is_idempotent_and_release_preserves_readers() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
let resource = crate::ObjectKey::new("bucket", "tail-downgrade");
|
||||
let first = request(resource.clone(), "owner-a", Duration::from_secs(30));
|
||||
assert!(client.acquire_lock(&first).await.expect("initial lock").success);
|
||||
assert!(client.downgrade_committed_tail(&first.lock_id).await.expect("downgrade"));
|
||||
assert!(
|
||||
client
|
||||
.downgrade_committed_tail(&first.lock_id)
|
||||
.await
|
||||
.expect("retry downgrade")
|
||||
);
|
||||
let reader = crate::LockRequest::new(resource.clone(), LockType::Shared, "reader")
|
||||
.with_acquire_timeout(Duration::from_millis(500));
|
||||
assert!(client.acquire_lock(&reader).await.expect("reader").success);
|
||||
let writer = request(resource, "writer", Duration::from_secs(30)).with_acquire_timeout(Duration::from_millis(10));
|
||||
assert!(!client.acquire_lock(&writer).await.expect("writer blocked").success);
|
||||
assert!(client.release(&first.lock_id).await.expect("release tail"));
|
||||
assert!(!client.acquire_lock(&writer).await.expect("reader excludes writer").success);
|
||||
assert!(client.release(&reader.lock_id).await.expect("release reader"));
|
||||
assert!(client.acquire_lock(&writer).await.expect("writer after drain").success);
|
||||
assert!(client.release(&writer.lock_id).await.expect("release writer"));
|
||||
assert!(!client.downgrade_committed_tail(&first.lock_id).await.expect("released lease"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn acquire_rejects_a_lock_id_for_another_resource() {
|
||||
let client = LocalClient::with_manager(Arc::new(GlobalLockManager::new()));
|
||||
|
||||
@@ -20,12 +20,6 @@ use async_trait::async_trait;
|
||||
use futures::future::join_all;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Marker exchanged in a refresh request and response for a committed-tail conversion.
|
||||
pub const COMMITTED_TAIL_CONVERSION_MARKER: &str = "rustfs-committed-tail-v1";
|
||||
pub const COMMITTED_TAIL_UPGRADE_MARKER: &str = "rustfs-committed-tail-upgrade-v1";
|
||||
pub const HEAL_LOCK_TRANSITION_PROBE_MARKER: &str = "rustfs-heal-lock-transition-probe-v1";
|
||||
pub const HEAL_WAITING_WRITER_MARKER: &str = "rustfs-heal-waiting-writer-v1";
|
||||
|
||||
/// Lock client trait
|
||||
#[async_trait]
|
||||
pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
@@ -54,27 +48,6 @@ pub trait LockClient: Send + Sync + std::fmt::Debug {
|
||||
/// Refresh lock
|
||||
async fn refresh(&self, lock_id: &LockId) -> Result<bool>;
|
||||
|
||||
/// Convert an exclusive lease to a reader-compatible committed-tail lease.
|
||||
/// Unsupported peers retain their original lease and report `false`.
|
||||
async fn downgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Try to restore exclusive ownership, retaining the writer fence on failure.
|
||||
async fn try_upgrade_committed_tail(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Check support without changing the lease mode.
|
||||
async fn supports_heal_lock_transition(&self, _lock_id: &LockId) -> Result<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Observe foreground writer contention, or return `None` when unsupported.
|
||||
async fn waiting_writer(&self, _lock_id: &LockId) -> Result<Option<bool>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
/// Force release lock
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool>;
|
||||
|
||||
|
||||
@@ -262,8 +262,6 @@ pub struct DistributedLockGuard {
|
||||
/// All underlying leases that should be refreshed and released with the guard.
|
||||
entries: Vec<HeldLockEntry>,
|
||||
lock_type: LockType,
|
||||
/// Shared and exclusive quorums must intersect before a tail may convert.
|
||||
committed_tail_quorums_intersect: bool,
|
||||
/// If true, Drop will not try to release (used if user manually released).
|
||||
disarmed: bool,
|
||||
/// Background lease task for renewable leases.
|
||||
@@ -279,13 +277,7 @@ impl DistributedLockGuard {
|
||||
/// - `entries` is the full list of underlying leases that should be refreshed and released
|
||||
/// with this guard.
|
||||
/// - `heartbeat`: lease TTL, optional renewal cadence, quorum, and diagnostics.
|
||||
fn new(
|
||||
lock_id: LockId,
|
||||
entries: Vec<HeldLockEntry>,
|
||||
lock_type: LockType,
|
||||
heartbeat: LockHeartbeatConfig,
|
||||
committed_tail_quorums_intersect: bool,
|
||||
) -> Self {
|
||||
fn new(lock_id: LockId, entries: Vec<HeldLockEntry>, lock_type: LockType, heartbeat: LockHeartbeatConfig) -> Self {
|
||||
record_lock_held_acquire(lock_type);
|
||||
let lock_lost = Arc::new(LockLostSignal::default());
|
||||
match Self::quorum_valid_until(&entries, heartbeat.refresh_quorum) {
|
||||
@@ -304,7 +296,6 @@ impl DistributedLockGuard {
|
||||
lock_id,
|
||||
entries,
|
||||
lock_type,
|
||||
committed_tail_quorums_intersect,
|
||||
disarmed: false,
|
||||
refresh_task,
|
||||
lock_lost,
|
||||
@@ -421,70 +412,6 @@ impl DistributedLockGuard {
|
||||
&self.lock_id
|
||||
}
|
||||
|
||||
/// Convert retained peer leases without ever releasing writer exclusion.
|
||||
/// A peer without conversion support keeps its exclusive lease. Partial
|
||||
/// conversion is safe: both lease modes continue to exclude writers.
|
||||
pub async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.downgrade_committed_tail(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
/// Require every retained lease to confirm exclusivity before committing.
|
||||
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || self.is_lock_lost() {
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.try_upgrade_committed_tail(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
pub async fn supports_heal_lock_transition(&self) -> bool {
|
||||
if self.disarmed || self.lock_type != LockType::Exclusive || !self.committed_tail_quorums_intersect || self.is_lock_lost()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let results = futures::future::join_all(
|
||||
self.entries
|
||||
.iter()
|
||||
.map(|entry| entry.client.supports_heal_lock_transition(&entry.lock_id)),
|
||||
)
|
||||
.await;
|
||||
!self.is_lock_lost() && results.iter().all(|result| matches!(result, Ok(true)))
|
||||
}
|
||||
|
||||
/// Observe contention on every held lease. Unknown peers prevent a false idle result.
|
||||
pub async fn waiting_writer(&self) -> Option<bool> {
|
||||
if self.disarmed || self.is_lock_lost() {
|
||||
return None;
|
||||
}
|
||||
let results =
|
||||
futures::future::join_all(self.entries.iter().map(|entry| entry.client.waiting_writer(&entry.lock_id))).await;
|
||||
if self.is_lock_lost() {
|
||||
return None;
|
||||
}
|
||||
if results.iter().any(|result| matches!(result, Ok(Some(true)))) {
|
||||
return Some(true);
|
||||
}
|
||||
results
|
||||
.iter()
|
||||
.all(|result| matches!(result, Ok(Some(false))))
|
||||
.then_some(false)
|
||||
}
|
||||
|
||||
/// Actual granted peer identities, which may differ from the logical guard ID.
|
||||
pub fn granted_lock_ids(&self) -> impl Iterator<Item = &LockId> {
|
||||
self.entries.iter().map(|entry| &entry.lock_id)
|
||||
@@ -672,7 +599,6 @@ impl DistributedLock {
|
||||
individual_locks,
|
||||
request.lock_type,
|
||||
heartbeat,
|
||||
self.read_quorum().saturating_add(required_quorum) > self.clients.len(),
|
||||
)))
|
||||
} else {
|
||||
// Check if it's a timeout or quorum failure
|
||||
|
||||
@@ -39,8 +39,6 @@ pub struct FastLockGuard {
|
||||
shard: Option<Arc<LockShard>>, // None when locks are disabled
|
||||
released: bool,
|
||||
disabled: bool, // True when locks are disabled globally
|
||||
committed_tail: bool,
|
||||
sealed_for_commit: bool,
|
||||
/// Unique ID for this guard instance to prevent double-release
|
||||
guard_id: u64,
|
||||
}
|
||||
@@ -56,8 +54,6 @@ impl FastLockGuard {
|
||||
shard: Some(shard),
|
||||
released: false,
|
||||
disabled: false,
|
||||
committed_tail: false,
|
||||
sealed_for_commit: false,
|
||||
guard_id,
|
||||
}
|
||||
}
|
||||
@@ -72,8 +68,6 @@ impl FastLockGuard {
|
||||
shard: None,
|
||||
released: false,
|
||||
disabled: true,
|
||||
committed_tail: false,
|
||||
sealed_for_commit: false,
|
||||
guard_id,
|
||||
}
|
||||
}
|
||||
@@ -93,83 +87,6 @@ impl FastLockGuard {
|
||||
&self.owner
|
||||
}
|
||||
|
||||
/// The physical acquisition was force-released before registration.
|
||||
pub(crate) fn abandon_unregistered(&mut self) {
|
||||
if !self.released {
|
||||
self.released = true;
|
||||
record_lock_held_release(self.mode);
|
||||
}
|
||||
}
|
||||
|
||||
/// Keep excluding writers while permitting reads of the committed version.
|
||||
pub fn downgrade_committed_tail(&mut self) -> bool {
|
||||
if self.released || self.sealed_for_commit {
|
||||
return false;
|
||||
}
|
||||
if self.committed_tail {
|
||||
return true;
|
||||
}
|
||||
if self.mode != LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
if self.disabled {
|
||||
self.mode = LockMode::Shared;
|
||||
self.committed_tail = true;
|
||||
return true;
|
||||
}
|
||||
let Some(shard) = &self.shard else { return false };
|
||||
if !shard.downgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
|
||||
return false;
|
||||
}
|
||||
record_lock_held_release(LockMode::Exclusive);
|
||||
record_lock_held_acquire(LockMode::Shared);
|
||||
self.mode = LockMode::Shared;
|
||||
self.committed_tail = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// Try to restore exclusive ownership without releasing the writer fence.
|
||||
pub fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
if self.released || (!self.committed_tail && self.mode != LockMode::Exclusive) {
|
||||
return false;
|
||||
}
|
||||
if !self.disabled {
|
||||
let Some(shard) = &self.shard else { return false };
|
||||
if !shard.try_upgrade_committed_tail(&self.key, &self.owner, self.guard_id) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if self.committed_tail {
|
||||
record_lock_held_release(LockMode::Shared);
|
||||
record_lock_held_acquire(LockMode::Exclusive);
|
||||
}
|
||||
self.mode = LockMode::Exclusive;
|
||||
self.committed_tail = false;
|
||||
self.sealed_for_commit = true;
|
||||
true
|
||||
}
|
||||
|
||||
/// Observe writers waiting on this guard's exact object key.
|
||||
pub fn waiting_writer(&self) -> Option<bool> {
|
||||
if self.released {
|
||||
return None;
|
||||
}
|
||||
if self.disabled {
|
||||
return Some(false);
|
||||
}
|
||||
self.shard
|
||||
.as_ref()?
|
||||
.waiting_writer_for_guard(&self.key, &self.owner, self.guard_id)
|
||||
}
|
||||
|
||||
fn release_physical(&self, shard: &LockShard) -> bool {
|
||||
if self.committed_tail {
|
||||
shard.release_committed_tail_with_guard(&self.key, &self.owner, self.guard_id)
|
||||
} else {
|
||||
shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Manually release the lock early
|
||||
///
|
||||
/// Returns true if the lock was successfully released, false if it was
|
||||
@@ -189,7 +106,7 @@ impl FastLockGuard {
|
||||
}
|
||||
|
||||
if let Some(shard) = &self.shard {
|
||||
let success = self.release_physical(shard);
|
||||
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
|
||||
if success {
|
||||
self.released = true;
|
||||
record_lock_held_release(self.mode);
|
||||
@@ -252,7 +169,7 @@ impl Drop for FastLockGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(shard) = &self.shard {
|
||||
if !self.released && !self.disabled {
|
||||
let success = self.release_physical(shard);
|
||||
let success = shard.release_lock_with_guard(&self.key, &self.owner, self.mode, self.guard_id);
|
||||
if !success {
|
||||
// For high-concurrency scenarios, this is likely due to:
|
||||
// 1. Lock was already released by another thread
|
||||
|
||||
@@ -96,14 +96,11 @@ impl FastObjectLockManager {
|
||||
/// Acquire object lock
|
||||
pub async fn acquire_lock(&self, request: ObjectLockRequest) -> Result<FastLockGuard, LockResult> {
|
||||
let shard = self.get_shard(&request.key);
|
||||
match shard.acquire_guarded(&request).await {
|
||||
Ok(acquired) => {
|
||||
let mut guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
|
||||
match shard.acquire_lock(&request).await {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key, request.mode, request.owner, shard.clone());
|
||||
// Register guard to prevent premature cleanup
|
||||
if !shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired) {
|
||||
guard.abandon_unregistered();
|
||||
return Err(LockResult::Timeout);
|
||||
}
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
Ok(guard)
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
@@ -204,15 +201,31 @@ impl FastObjectLockManager {
|
||||
let mut all_failed = Vec::new();
|
||||
let mut guards = Vec::new();
|
||||
|
||||
for (_, requests) in shard_groups {
|
||||
for (shard_id, requests) in shard_groups {
|
||||
let shard = self.shards[*shard_id].clone();
|
||||
|
||||
for request in requests {
|
||||
let key = request.key.clone();
|
||||
match self.acquire_lock(request.clone()).await {
|
||||
Ok(guard) => {
|
||||
all_successful.push(key);
|
||||
guards.push(guard);
|
||||
let owner = request.owner.clone();
|
||||
let mode = request.mode;
|
||||
|
||||
let acquired = if shard.try_fast_path_only(request) {
|
||||
true
|
||||
} else {
|
||||
match shard.acquire_lock(request).await {
|
||||
Ok(()) => true,
|
||||
Err(err) => {
|
||||
all_failed.push((key.clone(), err));
|
||||
false
|
||||
}
|
||||
}
|
||||
Err(err) => all_failed.push((key, err)),
|
||||
};
|
||||
|
||||
if acquired {
|
||||
let guard = FastLockGuard::new(key.clone(), mode, owner.clone(), shard.clone());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
all_successful.push(key);
|
||||
guards.push(guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -232,10 +245,14 @@ impl FastObjectLockManager {
|
||||
let mut acquired_guards = Vec::new();
|
||||
let mut failed_locks = Vec::new();
|
||||
|
||||
'outer: for (_, requests) in shard_groups {
|
||||
'outer: for (shard_id, requests) in shard_groups {
|
||||
let shard = self.shards[*shard_id].clone();
|
||||
|
||||
for request in requests {
|
||||
match self.acquire_lock(request.clone()).await {
|
||||
Ok(guard) => {
|
||||
match shard.acquire_lock(request).await {
|
||||
Ok(()) => {
|
||||
let guard = FastLockGuard::new(request.key.clone(), request.mode, request.owner.clone(), shard.clone());
|
||||
shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner());
|
||||
acquired_guards.push(guard);
|
||||
}
|
||||
Err(err) => {
|
||||
|
||||
@@ -94,7 +94,6 @@ impl ObjectLockState {
|
||||
pub fn reset_for_reuse(&mut self) {
|
||||
// Reset atomic state
|
||||
self.atomic_state = crate::fast_lock::state::AtomicLockState::new();
|
||||
self.force_epoch.store(0, Ordering::Relaxed);
|
||||
|
||||
// Clear owners
|
||||
*self.current_owner.write() = None;
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
use parking_lot::RwLock;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use tokio::time::timeout;
|
||||
|
||||
@@ -31,12 +30,6 @@ struct ActiveGuardInfo {
|
||||
key: ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: Arc<str>,
|
||||
committed_tail: bool,
|
||||
}
|
||||
|
||||
pub(crate) struct AcquiredLock {
|
||||
state: Arc<ObjectLockState>,
|
||||
force_epoch: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Hash)]
|
||||
@@ -53,8 +46,6 @@ enum TestPausePoint {
|
||||
BeforeFastWrite,
|
||||
FastWriteExisting,
|
||||
SlowLookup,
|
||||
ForceAfterTailRelease,
|
||||
ReleaseAfterUnregister,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -78,7 +69,7 @@ fn test_pause(point: TestPausePoint, key: &ObjectKey) {
|
||||
|
||||
/// Lock shard to reduce global contention.
|
||||
///
|
||||
/// When combined, lock `active_guards`, then `objects`, then a state's owner metadata.
|
||||
/// When both are needed, lock `objects` before a state's owner metadata.
|
||||
#[derive(Debug)]
|
||||
pub struct LockShard {
|
||||
/// Object lock states - using parking_lot for better performance
|
||||
@@ -142,16 +133,12 @@ impl LockShard {
|
||||
|
||||
/// Acquire lock with fast path optimization
|
||||
pub async fn acquire_lock(&self, request: &ObjectLockRequest) -> Result<(), LockResult> {
|
||||
self.acquire_guarded(request).await.map(|_| ())
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_guarded(&self, request: &ObjectLockRequest) -> Result<AcquiredLock, LockResult> {
|
||||
let start_time = Instant::now();
|
||||
|
||||
// Try fast path first
|
||||
if let Some(acquired) = self.try_fast_path(request) {
|
||||
if let Some(_state) = self.try_fast_path(request) {
|
||||
self.metrics.record_fast_path_success();
|
||||
return Ok(acquired);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Slow path with waiting
|
||||
@@ -170,7 +157,7 @@ impl LockShard {
|
||||
}
|
||||
|
||||
/// Try fast path lock acquisition (lock-free when possible)
|
||||
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<AcquiredLock> {
|
||||
fn try_fast_path(&self, request: &ObjectLockRequest) -> Option<Arc<ObjectLockState>> {
|
||||
// First try to get existing state without write lock
|
||||
{
|
||||
let objects = self.objects.read();
|
||||
@@ -181,16 +168,14 @@ impl LockShard {
|
||||
test_pause(TestPausePoint::FastExisting, &request.key);
|
||||
|
||||
// Try atomic acquisition
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let success = force_epoch != u64::MAX
|
||||
&& match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
let success = match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
drop(objects);
|
||||
|
||||
if success {
|
||||
return Some(AcquiredLock { state, force_epoch });
|
||||
return Some(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -210,11 +195,10 @@ impl LockShard {
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::FastWriteExisting, &request.key);
|
||||
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let acquired = force_epoch != u64::MAX && state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
|
||||
let acquired = state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout);
|
||||
drop(objects);
|
||||
if acquired {
|
||||
return Some(AcquiredLock { state, force_epoch });
|
||||
return Some(state);
|
||||
}
|
||||
} else {
|
||||
// Create new state from pool and acquire immediately
|
||||
@@ -222,7 +206,7 @@ impl LockShard {
|
||||
let state = Arc::new(*state_box);
|
||||
if state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout) {
|
||||
objects.insert(request.key.clone(), state.clone());
|
||||
return Some(AcquiredLock { state, force_epoch: 0 });
|
||||
return Some(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -231,7 +215,7 @@ impl LockShard {
|
||||
}
|
||||
|
||||
/// Slow path with async waiting
|
||||
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<AcquiredLock, LockResult> {
|
||||
async fn acquire_lock_slow_path(&self, request: &ObjectLockRequest, start_time: Instant) -> Result<(), LockResult> {
|
||||
// Use adaptive timeout based on current load and request priority
|
||||
let adaptive_timeout = self.calculate_adaptive_timeout(request);
|
||||
let deadline = start_time + adaptive_timeout;
|
||||
@@ -250,7 +234,7 @@ impl LockShard {
|
||||
|
||||
loop {
|
||||
// Keep the state in the map until its acquisition attempt completes.
|
||||
let (state, success, force_epoch) = {
|
||||
let (state, success) = {
|
||||
let mut objects = self.objects.write();
|
||||
let state = match objects.get(&request.key) {
|
||||
Some(state) => state.clone(),
|
||||
@@ -265,19 +249,17 @@ impl LockShard {
|
||||
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::SlowLookup, &request.key);
|
||||
let force_epoch = state.force_epoch.load(Ordering::Acquire);
|
||||
let success = force_epoch != u64::MAX
|
||||
&& match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
let success = match request.mode {
|
||||
LockMode::Shared => state.try_acquire_shared_fast(&request.owner, request.lock_timeout),
|
||||
LockMode::Exclusive => state.try_acquire_exclusive_fast(&request.owner, request.lock_timeout),
|
||||
};
|
||||
drop(objects);
|
||||
(state, success, force_epoch)
|
||||
(state, success)
|
||||
};
|
||||
|
||||
if success {
|
||||
self.metrics.record_slow_path_success();
|
||||
return Ok(AcquiredLock { state, force_epoch });
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Check timeout
|
||||
@@ -399,10 +381,14 @@ impl LockShard {
|
||||
|
||||
/// Release lock with guard ID tracking for double-release prevention
|
||||
pub fn release_lock_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, mode: LockMode, guard_id: u64) -> bool {
|
||||
// Keep force-unlock from releasing this holder and admitting a new
|
||||
// same-owner holder between guard removal and physical release.
|
||||
let mut guards = self.active_guards.lock();
|
||||
if guards.remove(&guard_id).is_none() {
|
||||
// First, try to remove the guard from active set
|
||||
let guard_was_active = {
|
||||
let mut guards = self.active_guards.lock();
|
||||
guards.remove(&guard_id).is_some()
|
||||
};
|
||||
|
||||
// If guard was not active, this is a double-release attempt
|
||||
if !guard_was_active {
|
||||
tracing::debug!(
|
||||
"Double-release attempt blocked: key={}, owner={}, mode={:?}, guard_id={}",
|
||||
key,
|
||||
@@ -413,9 +399,6 @@ impl LockShard {
|
||||
return false;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::ReleaseAfterUnregister, key);
|
||||
|
||||
// Proceed with normal release
|
||||
let should_cleanup;
|
||||
let result;
|
||||
@@ -440,8 +423,6 @@ impl LockShard {
|
||||
}
|
||||
}
|
||||
|
||||
drop(guards);
|
||||
|
||||
if should_cleanup {
|
||||
self.schedule_cleanup(key.clone());
|
||||
}
|
||||
@@ -449,110 +430,21 @@ impl LockShard {
|
||||
result
|
||||
}
|
||||
|
||||
/// Convert an active exclusive guard without opening a writer gap.
|
||||
pub(crate) fn downgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Exclusive {
|
||||
return false;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
if !state.downgrade_committed_tail(owner) {
|
||||
return false;
|
||||
}
|
||||
info.mode = LockMode::Shared;
|
||||
info.committed_tail = true;
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn try_upgrade_committed_tail(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get_mut(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
|
||||
return false;
|
||||
}
|
||||
if info.mode == LockMode::Exclusive && !info.committed_tail {
|
||||
return true;
|
||||
}
|
||||
if info.mode != LockMode::Shared || !info.committed_tail {
|
||||
return false;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
if !state.try_upgrade_committed_tail(owner) {
|
||||
return false;
|
||||
}
|
||||
info.mode = LockMode::Exclusive;
|
||||
info.committed_tail = false;
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn waiting_writer_for_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> Option<bool> {
|
||||
let guards = self.active_guards.lock();
|
||||
let info = guards.get(&guard_id)?.as_ref()?;
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() {
|
||||
return None;
|
||||
}
|
||||
let objects = self.objects.read();
|
||||
let state = objects.get(key)?;
|
||||
Some(state.atomic_state.writer_observed_during_tail())
|
||||
}
|
||||
|
||||
/// Release a converted guard and its special reader-admission state.
|
||||
pub(crate) fn release_committed_tail_with_guard(&self, key: &ObjectKey, owner: &Arc<str>, guard_id: u64) -> bool {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let Some(Some(info)) = guards.get(&guard_id) else { return false };
|
||||
if info.key != *key || info.owner.as_ref() != owner.as_ref() || info.mode != LockMode::Shared || !info.committed_tail {
|
||||
return false;
|
||||
}
|
||||
let (success, should_cleanup) = {
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else { return false };
|
||||
let success = state.release_committed_tail(owner);
|
||||
(success, success && !state.is_locked() && !state.atomic_state.has_waiters())
|
||||
};
|
||||
if success {
|
||||
guards.remove(&guard_id);
|
||||
self.metrics.record_release();
|
||||
}
|
||||
drop(guards);
|
||||
if should_cleanup {
|
||||
self.schedule_cleanup(key.clone());
|
||||
}
|
||||
success
|
||||
}
|
||||
|
||||
/// Register a guard to prevent premature cleanup
|
||||
pub fn register_guard(&self, guard_id: u64) {
|
||||
self.active_guards.lock().insert(guard_id, None);
|
||||
}
|
||||
|
||||
pub(crate) fn register_guard_with_info(
|
||||
&self,
|
||||
guard_id: u64,
|
||||
key: &ObjectKey,
|
||||
mode: LockMode,
|
||||
owner: &Arc<str>,
|
||||
acquired: &AcquiredLock,
|
||||
) -> bool {
|
||||
pub(crate) fn register_guard_with_info(&self, guard_id: u64, key: &ObjectKey, mode: LockMode, owner: &Arc<str>) {
|
||||
let mut guards = self.active_guards.lock();
|
||||
let objects = self.objects.read();
|
||||
if !objects.get(key).is_some_and(|state| {
|
||||
Arc::ptr_eq(state, &acquired.state) && state.force_epoch.load(Ordering::Acquire) == acquired.force_epoch
|
||||
}) {
|
||||
return false;
|
||||
}
|
||||
guards.insert(
|
||||
guard_id,
|
||||
Some(ActiveGuardInfo {
|
||||
key: key.clone(),
|
||||
mode,
|
||||
owner: owner.clone(),
|
||||
committed_tail: false,
|
||||
}),
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
/// Unregister a guard (called when guard is dropped)
|
||||
@@ -827,57 +719,27 @@ impl LockShard {
|
||||
/// Returns the number of owners that were released. Used by the admin
|
||||
/// force-unlock path to clear a stuck resource.
|
||||
pub fn force_release_all(&self, key: &ObjectKey) -> usize {
|
||||
// Serialize tail conversion and prevent new acquisitions while taking
|
||||
// and releasing the owner snapshot. A new tail must never be mistaken
|
||||
// for an ordinary shared owner by force unlock.
|
||||
let mut guards = self.active_guards.lock();
|
||||
let objects = self.objects.write();
|
||||
let Some(state) = objects.get(key) else { return 0 };
|
||||
// Acquisitions completed before this point but not yet registered
|
||||
// must not later create guards for the released generation.
|
||||
let _ = state
|
||||
.force_epoch
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |epoch| epoch.checked_add(1));
|
||||
let tail = guards.iter().find_map(|(&id, info)| {
|
||||
info.as_ref()
|
||||
.filter(|info| info.committed_tail && info.key == *key)
|
||||
.map(|info| (id, info.owner.clone()))
|
||||
});
|
||||
let mut released = 0;
|
||||
if let Some((id, owner)) = tail
|
||||
&& state.release_committed_tail(&owner)
|
||||
{
|
||||
guards.remove(&id);
|
||||
self.metrics.record_release();
|
||||
released += 1;
|
||||
}
|
||||
#[cfg(test)]
|
||||
test_pause(TestPausePoint::ForceAfterTailRelease, key);
|
||||
let mut owners_modes = Vec::new();
|
||||
if let Some(info) = state.current_owner.read().clone() {
|
||||
owners_modes.push((info.owner, LockMode::Exclusive, 1));
|
||||
}
|
||||
for entry in state.shared_owners.read().iter() {
|
||||
owners_modes.push((entry.owner.clone(), LockMode::Shared, entry.count));
|
||||
}
|
||||
for (owner, mode, count) in owners_modes {
|
||||
let mut owner_released = false;
|
||||
for _ in 0..count {
|
||||
let success = match mode {
|
||||
LockMode::Shared => state.release_shared(&owner),
|
||||
LockMode::Exclusive => state.release_exclusive(&owner),
|
||||
};
|
||||
if success {
|
||||
self.metrics.record_release();
|
||||
owner_released = true;
|
||||
}
|
||||
let owners_modes: Vec<(Arc<str>, LockMode)> = {
|
||||
let objects = self.objects.read();
|
||||
let Some(state) = objects.get(key) else {
|
||||
return 0;
|
||||
};
|
||||
let mut pairs = Vec::new();
|
||||
if let Some(info) = state.current_owner.read().clone() {
|
||||
pairs.push((info.owner, LockMode::Exclusive));
|
||||
}
|
||||
if owner_released {
|
||||
for entry in state.shared_owners.read().iter() {
|
||||
pairs.push((entry.owner.clone(), LockMode::Shared));
|
||||
}
|
||||
pairs
|
||||
};
|
||||
|
||||
let mut released = 0;
|
||||
for (owner, mode) in owners_modes {
|
||||
if self.release_lock(key, &owner, mode) {
|
||||
released += 1;
|
||||
}
|
||||
}
|
||||
// Old guard drops must not release a later holder with the same owner.
|
||||
guards.retain(|_, info| info.as_ref().is_none_or(|info| info.key != *key));
|
||||
released
|
||||
}
|
||||
|
||||
@@ -1117,315 +979,6 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn force_unlock_cannot_interleave_an_old_release_with_a_new_holder() {
|
||||
for reader in [false, true] {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-release-gap");
|
||||
let old = if reader {
|
||||
manager.acquire_read_lock(key.clone(), "same-owner").await
|
||||
} else {
|
||||
manager.acquire_write_lock(key.clone(), "same-owner").await
|
||||
}
|
||||
.expect("old holder");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let (entered, resume) = install_pause(TestPausePoint::ReleaseAfterUnregister, &key);
|
||||
let old_drop = tokio::task::spawn_blocking(move || drop(old));
|
||||
entered.recv_timeout(Duration::from_secs(5)).expect("old release paused");
|
||||
|
||||
let can_force = shard.active_guards.try_lock().is_some();
|
||||
let mut force = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
|
||||
};
|
||||
let mut new_guard = if can_force {
|
||||
assert_eq!(force.await.expect("force join"), 1);
|
||||
let mut guard = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer");
|
||||
if reader {
|
||||
assert!(guard.downgrade_committed_tail());
|
||||
}
|
||||
resume.send(()).expect("resume old release");
|
||||
old_drop.await.expect("old release join");
|
||||
guard
|
||||
} else {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut force).await.is_err());
|
||||
resume.send(()).expect("resume old release");
|
||||
old_drop.await.expect("old release join");
|
||||
assert_eq!(force.await.expect("force join"), 0);
|
||||
manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer")
|
||||
};
|
||||
if reader && !can_force {
|
||||
assert!(new_guard.downgrade_committed_tail());
|
||||
}
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"an old release must not unlock the new writer"
|
||||
);
|
||||
drop(new_guard);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn force_unlock_does_not_capture_a_new_committed_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-tail-race");
|
||||
let mut old = manager.acquire_write_lock(key.clone(), "old").await.expect("old writer");
|
||||
assert!(old.downgrade_committed_tail());
|
||||
let (entered, resume) = install_pause(TestPausePoint::ForceAfterTailRelease, &key);
|
||||
let force = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::task::spawn_blocking(move || manager.force_unlock(&key))
|
||||
};
|
||||
entered.recv_timeout(Duration::from_secs(5)).expect("force unlock paused");
|
||||
let mut next = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_write_lock(key, "new").await })
|
||||
};
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut next).await.is_err());
|
||||
resume.send(()).expect("resume force unlock");
|
||||
assert_eq!(force.await.expect("force unlock task"), 1);
|
||||
let mut new = next.await.expect("new writer task").expect("new writer");
|
||||
assert!(new.downgrade_committed_tail());
|
||||
drop(old);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"the new committed tail must still exclude writers"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_tail_upgrade_waits_for_readers_and_keeps_writer_fenced() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgrade-tail");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = manager
|
||||
.acquire_read_lock(key.clone(), "reader")
|
||||
.await
|
||||
.expect("reader during tail");
|
||||
assert!(!tail.try_upgrade_committed_tail(), "an active reader must prevent exclusive commit");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "late-reader"))
|
||||
.await
|
||||
.is_err(),
|
||||
"new readers must not starve a pending upgrade"
|
||||
);
|
||||
drop(reader);
|
||||
assert!(tail.try_upgrade_committed_tail());
|
||||
assert!(tail.try_upgrade_committed_tail(), "a repeated request must be idempotent");
|
||||
assert!(
|
||||
!tail.downgrade_committed_tail(),
|
||||
"a delayed downgrade must not reopen reads after upgrade"
|
||||
);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_read_lock(key.clone(), "reader"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
drop(tail);
|
||||
let _writer = manager.acquire_write_lock(key, "writer").await.expect("writer after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tail_remembers_a_writer_that_stopped_waiting_between_samples() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "brief-writer-observation");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(tail.waiting_writer(), Some(true), "a short wait must not be missed by polling");
|
||||
assert!(!tail.try_upgrade_committed_tail(), "observed contention must prevent a later commit");
|
||||
drop(tail);
|
||||
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
|
||||
assert_eq!(next.waiting_writer(), Some(false), "contention must not leak to the next owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upgraded_tail_remembers_a_writer_that_stopped_waiting() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgraded-tail-brief-writer");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert!(tail.try_upgrade_committed_tail());
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key.clone(), "writer"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(tail.waiting_writer(), Some(true), "upgraded leases must retain observed contention");
|
||||
drop(tail);
|
||||
let next = manager.acquire_write_lock(key, "next").await.expect("next lock");
|
||||
assert_eq!(next.waiting_writer(), Some(false));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn guard_observes_waiting_writer_and_rejects_stale_identity() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "waiting-writer-observation");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert_eq!(tail.waiting_writer(), Some(false));
|
||||
let writer = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_write_lock(key, "foreground").await })
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while tail.waiting_writer() != Some(true) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("writer becomes observable");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
assert_eq!(tail.waiting_writer(), None, "force-released guard must not inspect a replacement owner");
|
||||
let _writer = writer.await.expect("writer task").expect("writer succeeds after release");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dropping_a_pending_upgrade_allows_readers_again() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "cancel-upgrade");
|
||||
let mut tail = manager.acquire_write_lock(key.clone(), "heal").await.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = manager.acquire_read_lock(key.clone(), "reader").await.expect("first reader");
|
||||
assert!(!tail.try_upgrade_committed_tail());
|
||||
let mut waiting = {
|
||||
let manager = manager.clone();
|
||||
let key = key.clone();
|
||||
tokio::spawn(async move { manager.acquire_read_lock(key, "waiting-reader").await })
|
||||
};
|
||||
assert!(tokio::time::timeout(Duration::from_millis(100), &mut waiting).await.is_err());
|
||||
drop(tail);
|
||||
let second = tokio::time::timeout(Duration::from_secs(2), waiting)
|
||||
.await
|
||||
.expect("reader wakes after cancellation")
|
||||
.expect("reader task joins")
|
||||
.expect("reader obtains lock");
|
||||
drop((reader, second));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_released_tail_cannot_claim_an_exclusive_upgrade() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "upgrade-stale");
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("write lock");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let _new = manager.acquire_write_lock(key, "same-owner").await.expect("new owner");
|
||||
assert!(!tail.try_upgrade_committed_tail());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exclusive_upgrade_ack_fences_a_late_downgrade() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "late-downgrade");
|
||||
let mut guard = manager.acquire_write_lock(key, "heal").await.expect("write lock");
|
||||
assert!(guard.try_upgrade_committed_tail());
|
||||
assert!(!guard.downgrade_committed_tail());
|
||||
assert!(guard.try_upgrade_committed_tail());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_old_readers_cannot_release_a_new_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-old-readers");
|
||||
let old1 = manager
|
||||
.acquire_read_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("first reader");
|
||||
let old2 = manager
|
||||
.acquire_read_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("second reader");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), "same-owner")
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
drop((old1, old2));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"old reader drops must not release the new tail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn forced_unregistered_reader_cannot_release_a_new_tail() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "force-unregistered-reader");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let owner: Arc<str> = Arc::from("same-owner");
|
||||
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
|
||||
let acquired = shard
|
||||
.acquire_guarded(&request)
|
||||
.await
|
||||
.expect("old reader acquired before registration");
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let mut old = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
|
||||
assert!(!shard.register_guard_with_info(old.guard_id(), old.key(), old.mode(), old.owner(), &acquired));
|
||||
old.abandon_unregistered();
|
||||
drop(old);
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, "competitor"))
|
||||
.await
|
||||
.is_err(),
|
||||
"an old reader must not release a new committed tail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_on_another_key_keeps_unregistered_acquisition_valid() {
|
||||
let manager = test_manager();
|
||||
let key = ObjectKey::new("bucket", "unrelated-acquisition");
|
||||
let shard = manager.get_shard(&key).clone();
|
||||
let owner: Arc<str> = Arc::from("owner");
|
||||
let request = ObjectLockRequest::new_read(key.clone(), owner.clone());
|
||||
let acquired = shard.acquire_guarded(&request).await.expect("reader acquired");
|
||||
let other = (0..1000)
|
||||
.map(|index| ObjectKey::new("bucket", format!("other-{index}")))
|
||||
.find(|other| Arc::ptr_eq(&shard, manager.get_shard(other)))
|
||||
.expect("another key on the same shard");
|
||||
let other_guard = manager
|
||||
.acquire_write_lock(other.clone(), "other")
|
||||
.await
|
||||
.expect("other writer");
|
||||
assert_eq!(manager.force_unlock(&other), 1);
|
||||
drop(other_guard);
|
||||
let guard = FastLockGuard::new(key.clone(), LockMode::Shared, owner, shard.clone());
|
||||
assert!(shard.register_guard_with_info(guard.guard_id(), guard.key(), guard.mode(), guard.owner(), &acquired));
|
||||
drop(guard);
|
||||
assert!(manager.acquire_write_lock(key, "next").await.is_ok());
|
||||
}
|
||||
|
||||
fn start_acquisition(
|
||||
manager: &Arc<FastObjectLockManager>,
|
||||
request: ObjectLockRequest,
|
||||
|
||||
@@ -23,11 +23,6 @@ use crate::fast_lock::types::{LockMode, LockPriority};
|
||||
/// Optimized atomic lock state encoding in u64
|
||||
/// Bits: [63:48] reserved | [47:32] writers_waiting | [31:16] readers_waiting | [15:8] readers_count | [7:1] flags | [0] writer_flag
|
||||
const WRITER_FLAG_MASK: u64 = 0x1;
|
||||
// A committed write tail is a reader-compatible owner that still excludes writers.
|
||||
const COMMITTED_TAIL_FLAG: u64 = 0x2;
|
||||
const TAIL_UPGRADE_PENDING_FLAG: u64 = 0x4;
|
||||
const TAIL_WRITER_OBSERVED_FLAG: u64 = 0x8;
|
||||
const TAIL_TRACK_WRITERS_FLAG: u64 = 0x10;
|
||||
const READERS_SHIFT: u8 = 8;
|
||||
const READERS_MASK: u64 = 0xFF << READERS_SHIFT; // Support up to 255 concurrent readers
|
||||
const READERS_WAITING_SHIFT: u8 = 16;
|
||||
@@ -36,6 +31,7 @@ const WRITERS_WAITING_SHIFT: u8 = 32;
|
||||
const WRITERS_WAITING_MASK: u64 = 0xFFFF << WRITERS_WAITING_SHIFT;
|
||||
|
||||
// Fast path check masks
|
||||
const NO_WRITER_AND_NO_WAITING_WRITERS: u64 = WRITER_FLAG_MASK | WRITERS_WAITING_MASK;
|
||||
const COMPLETELY_UNLOCKED: u64 = 0;
|
||||
|
||||
/// Fast atomic lock state for single version
|
||||
@@ -70,8 +66,8 @@ impl AtomicLockState {
|
||||
let state = self.state.load(Ordering::Relaxed); // Use Relaxed for better performance
|
||||
match mode {
|
||||
LockMode::Shared => {
|
||||
(state & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) == 0
|
||||
&& ((state & WRITERS_WAITING_MASK) == 0 || (state & COMMITTED_TAIL_FLAG) != 0)
|
||||
// No writer and no waiting writers
|
||||
(state & NO_WRITER_AND_NO_WAITING_WRITERS) == 0
|
||||
}
|
||||
LockMode::Exclusive => {
|
||||
// Completely unlocked
|
||||
@@ -87,11 +83,8 @@ impl AtomicLockState {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
|
||||
// While a committed tail owns a shared slot, let reads through
|
||||
// even if a subsequent writer is waiting for that tail to drain.
|
||||
if (current & (WRITER_FLAG_MASK | TAIL_UPGRADE_PENDING_FLAG)) != 0
|
||||
|| ((current & WRITERS_WAITING_MASK) != 0 && (current & COMMITTED_TAIL_FLAG) == 0)
|
||||
{
|
||||
// Fast path check - cannot acquire if there's a writer or writers waiting
|
||||
if (current & NO_WRITER_AND_NO_WAITING_WRITERS) != 0 {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -144,78 +137,6 @@ impl AtomicLockState {
|
||||
}
|
||||
}
|
||||
|
||||
/// Atomically retain writer exclusion while allowing reads of a committed tail.
|
||||
fn downgrade_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & WRITER_FLAG_MASK) == 0 || (current & (READERS_MASK | COMMITTED_TAIL_FLAG)) != 0 {
|
||||
return false;
|
||||
}
|
||||
let next = (current & !WRITER_FLAG_MASK)
|
||||
| COMMITTED_TAIL_FLAG
|
||||
| (1 << READERS_SHIFT)
|
||||
| if current & WRITERS_WAITING_MASK != 0 {
|
||||
TAIL_WRITER_OBSERVED_FLAG
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Restore exclusive ownership only after every other reader has left.
|
||||
fn try_upgrade_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & COMMITTED_TAIL_FLAG) == 0 || (current & TAIL_WRITER_OBSERVED_FLAG) != 0 {
|
||||
return false;
|
||||
}
|
||||
let readers_drained = (current & READERS_MASK) == (1 << READERS_SHIFT);
|
||||
let next = if readers_drained {
|
||||
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG | READERS_MASK))
|
||||
| WRITER_FLAG_MASK
|
||||
| TAIL_TRACK_WRITERS_FLAG
|
||||
} else {
|
||||
current | TAIL_UPGRADE_PENDING_FLAG
|
||||
};
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return readers_drained;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Release the committed tail's own shared slot and restore ordinary fairness.
|
||||
fn release_committed_tail(&self) -> bool {
|
||||
loop {
|
||||
let current = self.state.load(Ordering::Acquire);
|
||||
if (current & COMMITTED_TAIL_FLAG) == 0 || self.readers_count(current) == 0 {
|
||||
return false;
|
||||
}
|
||||
let next =
|
||||
(current & !(COMMITTED_TAIL_FLAG | TAIL_UPGRADE_PENDING_FLAG | TAIL_WRITER_OBSERVED_FLAG)) - (1 << READERS_SHIFT);
|
||||
if self
|
||||
.state
|
||||
.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
self.update_access_time();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Release shared lock
|
||||
pub fn release_shared(&self) -> bool {
|
||||
loop {
|
||||
@@ -248,7 +169,7 @@ impl AtomicLockState {
|
||||
return false; // No exclusive lock to release
|
||||
}
|
||||
|
||||
let new_state = current & !(WRITER_FLAG_MASK | TAIL_TRACK_WRITERS_FLAG | TAIL_WRITER_OBSERVED_FLAG);
|
||||
let new_state = current & !WRITER_FLAG_MASK;
|
||||
|
||||
if self
|
||||
.state
|
||||
@@ -315,10 +236,7 @@ impl AtomicLockState {
|
||||
return false; // Max waiting writers
|
||||
}
|
||||
|
||||
let mut new_state = current + (1 << WRITERS_WAITING_SHIFT);
|
||||
if current & (COMMITTED_TAIL_FLAG | TAIL_TRACK_WRITERS_FLAG) != 0 {
|
||||
new_state |= TAIL_WRITER_OBSERVED_FLAG;
|
||||
}
|
||||
let new_state = current + (1 << WRITERS_WAITING_SHIFT);
|
||||
|
||||
if self
|
||||
.state
|
||||
@@ -403,15 +321,10 @@ impl AtomicLockState {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn writers_waiting_count(&self) -> u16 {
|
||||
pub fn writers_waiting_count(&self) -> u16 {
|
||||
let state = self.state.load(Ordering::Acquire);
|
||||
self.writers_waiting(state)
|
||||
}
|
||||
|
||||
pub(crate) fn writer_observed_during_tail(&self) -> bool {
|
||||
let state = self.state.load(Ordering::Acquire);
|
||||
self.writers_waiting(state) > 0 || state & TAIL_WRITER_OBSERVED_FLAG != 0
|
||||
}
|
||||
}
|
||||
|
||||
/// Object lock state with version support - optimized memory layout
|
||||
@@ -421,8 +334,6 @@ pub struct ObjectLockState {
|
||||
// First cache line: Most frequently accessed data
|
||||
/// Atomic state for fast operations
|
||||
pub atomic_state: AtomicLockState,
|
||||
/// Changes whenever an administrator force-releases this state.
|
||||
pub(crate) force_epoch: AtomicU64,
|
||||
|
||||
// Second cache line: Notification mechanisms
|
||||
/// Notification for readers (traditional)
|
||||
@@ -466,7 +377,6 @@ impl ObjectLockState {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
atomic_state: AtomicLockState::new(),
|
||||
force_epoch: AtomicU64::new(0),
|
||||
read_notify: Notify::new(),
|
||||
write_notify: Notify::new(),
|
||||
optimized_notify: OptimizedNotify::new(),
|
||||
@@ -515,70 +425,6 @@ impl ObjectLockState {
|
||||
true
|
||||
}
|
||||
|
||||
/// Convert the current exclusive owner into the committed-tail owner.
|
||||
pub(crate) fn downgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut current = self.current_owner.write();
|
||||
let Some(info) = current.as_ref().filter(|info| info.owner.as_ref() == owner.as_ref()).cloned() else {
|
||||
return false;
|
||||
};
|
||||
let mut shared = self.shared_owners.write();
|
||||
if !shared.is_empty() || !self.atomic_state.downgrade_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
shared.push(SharedOwnerEntry {
|
||||
owner: info.owner,
|
||||
count: 1,
|
||||
acquired_at: info.acquired_at,
|
||||
lock_timeout: info.lock_timeout,
|
||||
});
|
||||
*current = None;
|
||||
drop(shared);
|
||||
drop(current);
|
||||
self.optimized_notify.notify_readers();
|
||||
true
|
||||
}
|
||||
|
||||
pub(crate) fn try_upgrade_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut current = self.current_owner.write();
|
||||
let mut shared = self.shared_owners.write();
|
||||
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
|
||||
return false;
|
||||
};
|
||||
if current.is_some() || !self.atomic_state.try_upgrade_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
let info = shared.remove(pos);
|
||||
*current = Some(ExclusiveOwnerInfo {
|
||||
owner: info.owner,
|
||||
acquired_at: info.acquired_at,
|
||||
lock_timeout: info.lock_timeout,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
/// Release the committed-tail owner's shared slot without releasing readers.
|
||||
pub(crate) fn release_committed_tail(&self, owner: &Arc<str>) -> bool {
|
||||
let mut shared = self.shared_owners.write();
|
||||
let Some(pos) = shared.iter().position(|entry| entry.owner.as_ref() == owner.as_ref()) else {
|
||||
return false;
|
||||
};
|
||||
if !self.atomic_state.release_committed_tail() {
|
||||
return false;
|
||||
}
|
||||
if shared[pos].count > 1 {
|
||||
shared[pos].count -= 1;
|
||||
} else {
|
||||
shared.remove(pos);
|
||||
}
|
||||
let no_readers = shared.is_empty();
|
||||
drop(shared);
|
||||
if no_readers {
|
||||
self.optimized_notify.notify_writer();
|
||||
}
|
||||
self.optimized_notify.notify_readers();
|
||||
true
|
||||
}
|
||||
|
||||
/// Release shared lock
|
||||
pub fn release_shared(&self, owner: &Arc<str>) -> bool {
|
||||
let mut shared = self.shared_owners.write();
|
||||
@@ -709,24 +555,6 @@ mod tests {
|
||||
assert!(!state.release_exclusive());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn committed_tail_allows_reads_but_preserves_writer_exclusion() {
|
||||
let state = AtomicLockState::new();
|
||||
assert!(state.try_acquire_exclusive());
|
||||
assert!(state.inc_writers_waiting());
|
||||
assert!(state.downgrade_committed_tail());
|
||||
assert!(!state.try_acquire_exclusive(), "the tail must still exclude writers");
|
||||
assert!(state.try_acquire_shared(), "a queued writer must not block reads of the committed tail");
|
||||
assert!(state.release_committed_tail());
|
||||
assert!(!state.try_acquire_shared(), "writer preference resumes when the tail ends");
|
||||
assert!(!state.try_acquire_exclusive(), "an outstanding reader must still exclude writers");
|
||||
assert!(state.release_shared());
|
||||
assert!(state.try_acquire_exclusive());
|
||||
assert!(state.release_exclusive());
|
||||
state.dec_writers_waiting();
|
||||
assert!(state.is_free());
|
||||
}
|
||||
|
||||
// Regression for the waiter-preserving exclusive CAS.
|
||||
//
|
||||
// The acquisition CAS used to demand a fully-zero state word, which
|
||||
|
||||
@@ -111,67 +111,6 @@ mod fast_lock_tests {
|
||||
assert!(guard.is_released(), "Guard should be marked as released");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn force_unlock_old_tail_cannot_release_new_tail() {
|
||||
let manager = create_test_manager();
|
||||
let key = ObjectKey::new("test-bucket", "force-tail-generation");
|
||||
let owner: Arc<str> = Arc::from("same-owner");
|
||||
let mut old = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("old writer");
|
||||
assert!(old.downgrade_committed_tail());
|
||||
assert_eq!(manager.force_unlock(&key), 1);
|
||||
let mut new = manager
|
||||
.acquire_write_lock(key.clone(), owner.clone())
|
||||
.await
|
||||
.expect("new writer");
|
||||
assert!(new.downgrade_committed_tail());
|
||||
drop(old);
|
||||
let attempt =
|
||||
tokio::time::timeout(Duration::from_millis(100), manager.acquire_write_lock(key, Arc::from("competing-writer")))
|
||||
.await;
|
||||
assert!(attempt.is_err(), "dropping the old guard must not release the new tail");
|
||||
drop(new);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn committed_tail_allows_reader_while_writer_waits() {
|
||||
let manager = Arc::new(create_test_manager());
|
||||
let key = ObjectKey::new("test-bucket", "tail-handoff");
|
||||
let mut tail = manager
|
||||
.acquire_write_lock(key.clone(), Arc::from("first-writer"))
|
||||
.await
|
||||
.expect("initial writer");
|
||||
let writer = tokio::spawn({
|
||||
let manager = Arc::clone(&manager);
|
||||
let key = key.clone();
|
||||
async move { manager.acquire_write_lock(key, Arc::from("second-writer")).await }
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while manager.current_lock_counts().1 == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("second writer should be queued");
|
||||
assert!(tail.downgrade_committed_tail());
|
||||
let reader = tokio::time::timeout(Duration::from_secs(2), manager.acquire_read_lock(key, Arc::from("reader")))
|
||||
.await
|
||||
.expect("reader should not wait behind committed tail")
|
||||
.expect("reader should acquire");
|
||||
assert!(!writer.is_finished(), "tail and reader must exclude the next writer");
|
||||
drop(tail);
|
||||
assert!(!writer.is_finished(), "reader still excludes the next writer");
|
||||
drop(reader);
|
||||
let writer = tokio::time::timeout(Duration::from_secs(2), writer)
|
||||
.await
|
||||
.expect("writer should resume")
|
||||
.expect("writer task should join")
|
||||
.expect("writer should acquire");
|
||||
drop(writer);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_lock_counts_report_live_holders_and_waiters() {
|
||||
let manager = Arc::new(create_test_manager());
|
||||
|
||||
@@ -107,38 +107,6 @@ impl NamespaceLockGuard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Permit reads while continuing to exclude competing writers.
|
||||
pub async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.downgrade_committed_tail().await,
|
||||
Self::Fast(guard) => guard.downgrade_committed_tail(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to restore exclusive access while retaining the writer fence.
|
||||
pub async fn try_upgrade_committed_tail(&mut self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.try_upgrade_committed_tail().await,
|
||||
Self::Fast(guard) => guard.try_upgrade_committed_tail(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether the held leases can safely use heal lock transitions.
|
||||
pub async fn supports_heal_lock_transition(&self) -> bool {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.supports_heal_lock_transition().await,
|
||||
Self::Fast(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe whether a writer is waiting on this object's held leases.
|
||||
pub async fn waiting_writer(&self) -> Option<bool> {
|
||||
match self {
|
||||
Self::Standard(guard) => guard.waiting_writer().await,
|
||||
Self::Fast(guard) => guard.waiting_writer(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Manually release the lock early
|
||||
pub fn release(&mut self) -> bool {
|
||||
match self {
|
||||
|
||||
@@ -154,43 +154,6 @@ impl crate::client::LockClient for DelayedClient {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PendingTailClient(LocalClient);
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::client::LockClient for PendingTailClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> crate::Result<LockResponse> {
|
||||
self.0.acquire_lock(request).await
|
||||
}
|
||||
async fn release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.release(lock_id).await
|
||||
}
|
||||
async fn refresh(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.refresh(lock_id).await
|
||||
}
|
||||
async fn downgrade_committed_tail(&self, _: &LockId) -> crate::Result<bool> {
|
||||
std::future::pending().await
|
||||
}
|
||||
async fn force_release(&self, lock_id: &LockId) -> crate::Result<bool> {
|
||||
self.0.force_release(lock_id).await
|
||||
}
|
||||
async fn check_status(&self, lock_id: &LockId) -> crate::Result<Option<LockInfo>> {
|
||||
self.0.check_status(lock_id).await
|
||||
}
|
||||
async fn get_stats(&self) -> crate::Result<LockStats> {
|
||||
self.0.get_stats().await
|
||||
}
|
||||
async fn close(&self) -> crate::Result<()> {
|
||||
self.0.close().await
|
||||
}
|
||||
async fn is_online(&self) -> bool {
|
||||
self.0.is_online().await
|
||||
}
|
||||
async fn is_local(&self) -> bool {
|
||||
self.0.is_local().await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct FlakyAcquireClient {
|
||||
inner: LocalClient,
|
||||
@@ -616,286 +579,6 @@ async fn test_namespace_lock_guard_lock_id() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tail_conversion_rejects_non_intersecting_quorums() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..4)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients_and_quorum("non-intersecting".to_owned(), clients, 2);
|
||||
let resource = ObjectKey::new("bucket", "non-intersecting-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("write lease");
|
||||
assert!(
|
||||
!tail.downgrade_committed_tail().await,
|
||||
"conversion requires intersecting read and write quorums"
|
||||
);
|
||||
assert!(tail.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_tail_conversion_keeps_writers_excluded() {
|
||||
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = Vec::new();
|
||||
for index in 0..4 {
|
||||
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
if index < 2 {
|
||||
clients.push(inner);
|
||||
} else {
|
||||
clients.push(Arc::new(DelayedClient {
|
||||
inner,
|
||||
delay: Duration::ZERO,
|
||||
}));
|
||||
}
|
||||
}
|
||||
let lock = NamespaceLock::with_clients("mixed-peers".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "partial-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(!tail.downgrade_committed_tail().await, "mixed peers cannot all confirm conversion");
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"mixed leases still exclude writers"
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_tail_conversion_keeps_writers_excluded() {
|
||||
let mut clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
clients.push(Arc::new(PendingTailClient(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))));
|
||||
let lock = NamespaceLock::with_clients_and_quorum("cancel-tail".to_owned(), clients, 4);
|
||||
let resource = ObjectKey::new("bucket", "cancel-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), tail.downgrade_committed_tail())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("other peers converted before cancellation");
|
||||
assert!(reader.release());
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"cancelled conversion must still exclude writers"
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unsupported_peer_keeps_exclusive_tail_lease() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| {
|
||||
let inner = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
|
||||
Arc::new(DelayedClient {
|
||||
inner,
|
||||
delay: Duration::ZERO,
|
||||
}) as Arc<dyn crate::client::LockClient>
|
||||
})
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("old-peers".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "old-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(!tail.supports_heal_lock_transition().await, "old peers must not claim support");
|
||||
assert_eq!(tail.waiting_writer().await, None, "old peers cannot report contention");
|
||||
assert!(!tail.downgrade_committed_tail().await, "old peers must not claim conversion");
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "old peers must not claim upgrade support");
|
||||
assert!(
|
||||
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "writer", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after tail");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_committed_tail_allows_reader_and_excludes_writer() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("tail-test".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "committed-tail");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "first", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("first writer");
|
||||
assert!(tail.downgrade_committed_tail().await, "all held peer leases should convert");
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("committed reader");
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"tail and reader must exclude writer"
|
||||
);
|
||||
assert!(tail.release());
|
||||
assert!(
|
||||
lock.get_write_lock(resource.clone(), "second", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err(),
|
||||
"reader still excludes writer"
|
||||
);
|
||||
assert!(reader.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "second", Duration::from_secs(3))
|
||||
.await
|
||||
.expect("writer after drain");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_tail_upgrade_requires_all_leases_and_drained_readers() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients("tail-upgrade".to_owned(), clients);
|
||||
let resource = ObjectKey::new("bucket", "tail-upgrade");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("write lock");
|
||||
assert!(tail.supports_heal_lock_transition().await);
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
let mut reader = lock
|
||||
.get_read_lock(resource.clone(), "reader", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("reader");
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "live readers prevent full upgrade");
|
||||
assert!(reader.release());
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while !tail.try_upgrade_committed_tail().await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("retry must finish partial upgrades after asynchronous reader release");
|
||||
assert!(tail.try_upgrade_committed_tail().await, "upgrade retries must be idempotent");
|
||||
assert!(
|
||||
lock.get_read_lock(resource.clone(), "reader", Duration::from_millis(150))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(tail.release());
|
||||
let mut writer = lock
|
||||
.get_write_lock(resource, "writer", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("writer after release");
|
||||
assert!(writer.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_tail_observes_a_waiting_writer() {
|
||||
let clients: Vec<Arc<dyn crate::client::LockClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))) as Arc<dyn crate::client::LockClient>)
|
||||
.collect();
|
||||
let lock = Arc::new(NamespaceLock::with_clients("writer-observation".to_owned(), clients));
|
||||
let resource = ObjectKey::new("bucket", "writer-observation");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("heal lock");
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
assert_eq!(tail.waiting_writer().await, Some(false));
|
||||
let writer = {
|
||||
let lock = Arc::clone(&lock);
|
||||
let resource = resource.clone();
|
||||
tokio::spawn(async move { lock.get_write_lock(resource, "writer", Duration::from_secs(3)).await })
|
||||
};
|
||||
tokio::time::timeout(Duration::from_secs(2), async {
|
||||
while tail.waiting_writer().await != Some(true) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("waiting writer becomes visible on a held peer");
|
||||
assert!(tail.release());
|
||||
let mut guard = writer.await.expect("writer task").expect("writer succeeds after release");
|
||||
assert!(guard.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_partial_upgrade_remembers_contention_on_upgraded_peer() {
|
||||
use crate::client::LockClient;
|
||||
|
||||
let clients: Vec<Arc<LocalClient>> = (0..3)
|
||||
.map(|_| Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new()))))
|
||||
.collect();
|
||||
let lock = NamespaceLock::with_clients(
|
||||
"partial-upgrade-writer".to_owned(),
|
||||
clients.iter().cloned().map(|client| client as Arc<dyn LockClient>).collect(),
|
||||
);
|
||||
let resource = ObjectKey::new("bucket", "partial-upgrade-writer");
|
||||
let mut tail = lock
|
||||
.get_write_lock(resource.clone(), "heal", Duration::from_secs(2))
|
||||
.await
|
||||
.expect("heal lock");
|
||||
let ids = match &tail {
|
||||
NamespaceLockGuard::Standard(guard) => guard.granted_lock_ids().cloned().collect::<Vec<_>>(),
|
||||
NamespaceLockGuard::Fast(_) => panic!("distributed guard expected"),
|
||||
};
|
||||
let mut holders = Vec::new();
|
||||
for client in &clients {
|
||||
for id in &ids {
|
||||
if client.check_status(id).await.expect("peer status").is_some() {
|
||||
holders.push(Arc::clone(client));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(holders.len() >= 2, "write quorum must hold multiple peers");
|
||||
assert!(tail.downgrade_committed_tail().await);
|
||||
let reader = LockRequest::new(resource.clone(), LockType::Shared, "reader");
|
||||
assert!(holders[0].acquire_lock(&reader).await.expect("reader request").success);
|
||||
assert!(!tail.try_upgrade_committed_tail().await, "reader prevents a full upgrade");
|
||||
let writer = LockRequest::new(resource, LockType::Exclusive, "writer").with_acquire_timeout(Duration::from_millis(100));
|
||||
assert!(!holders[1].acquire_lock(&writer).await.expect("writer request").success);
|
||||
assert_eq!(
|
||||
tail.waiting_writer().await,
|
||||
Some(true),
|
||||
"partial upgrade must retain the earlier contention"
|
||||
);
|
||||
assert!(holders[0].release(&reader.lock_id).await.expect("release reader"));
|
||||
assert!(tail.release());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_lock_distributed_multi_node_simulation() {
|
||||
// Simulate a 3-node distributed environment where each node has its own lock backend
|
||||
|
||||
@@ -30,6 +30,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-tls-runtime?/fips"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
@@ -110,6 +110,9 @@ where
|
||||
ftps_required = self.config.ftps_required,
|
||||
"FTPS server starting"
|
||||
);
|
||||
if self.config.tls_enabled && self.config.cert_dir.is_some() {
|
||||
rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| FtpsInitError::InvalidConfig(err.to_string()))?;
|
||||
}
|
||||
let (reload_shutdown_tx, reload_shutdown_rx) = watch::channel(false);
|
||||
|
||||
let storage_clone = self.storage.clone();
|
||||
@@ -191,8 +194,6 @@ where
|
||||
);
|
||||
|
||||
// Build ServerConfig with SNI support
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
let server_config = rustls::ServerConfig::builder()
|
||||
.with_no_client_auth()
|
||||
.with_cert_resolver(resolver);
|
||||
|
||||
@@ -43,3 +43,70 @@ pub use webdav::{config::WebDavConfig, server::WebDavServer};
|
||||
|
||||
#[cfg(feature = "sftp")]
|
||||
pub use sftp::{SftpConfig, SftpInitError, SftpServer};
|
||||
|
||||
#[cfg(all(test, feature = "fips", feature = "ftps", feature = "webdav"))]
|
||||
mod fips_tests {
|
||||
use crate::common::dummy_storage::DummyBackend;
|
||||
use crate::{FtpsConfig, FtpsServer, WebDavConfig, WebDavServer};
|
||||
|
||||
#[test]
|
||||
fn protocols_reject_incompatible_host_provider() {
|
||||
const CHILD: &str = "RUSTFS_TEST_PROTOCOL_FIPS_PROVIDER";
|
||||
if std::env::var_os(CHILD).is_some() {
|
||||
let mut provider = rustfs_tls_runtime::default_crypto_provider();
|
||||
provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
|
||||
provider.install_default().expect("install incompatible host provider");
|
||||
let cert_dir = tempfile::tempdir().expect("certificate directory");
|
||||
let cert_dir = cert_dir.path().to_string_lossy().into_owned();
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
runtime.block_on(async {
|
||||
let ftps_config = FtpsConfig {
|
||||
cert_dir: Some(cert_dir.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let ftps = FtpsServer::new(ftps_config, DummyBackend::default())
|
||||
.await
|
||||
.expect("FTPS config");
|
||||
let (_sender, receiver) = tokio::sync::broadcast::channel(1);
|
||||
let error = ftps
|
||||
.start(receiver)
|
||||
.await
|
||||
.expect_err("FTPS must reject the incompatible provider");
|
||||
assert!(error.to_string().contains("process-wide rustls crypto provider"));
|
||||
|
||||
let webdav_config = WebDavConfig {
|
||||
bind_addr: ([127, 0, 0, 1], 0).into(),
|
||||
cert_dir: Some(cert_dir),
|
||||
..Default::default()
|
||||
};
|
||||
let webdav = WebDavServer::new(webdav_config, DummyBackend::default())
|
||||
.await
|
||||
.expect("WebDAV config");
|
||||
let (_sender, receiver) = tokio::sync::broadcast::channel(1);
|
||||
let error = webdav
|
||||
.start(receiver)
|
||||
.await
|
||||
.expect_err("WebDAV must reject the incompatible provider");
|
||||
assert!(error.to_string().contains("process-wide rustls crypto provider"));
|
||||
});
|
||||
return;
|
||||
}
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.arg("protocols_reject_incompatible_host_provider")
|
||||
.arg("--nocapture")
|
||||
.env(CHILD, "1")
|
||||
.env("NO_PROXY", "localhost,127.0.0.1,::1")
|
||||
.env("no_proxy", "localhost,127.0.0.1,::1")
|
||||
.output()
|
||||
.expect("start isolated protocol TLS test");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"stdout={} stderr={}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -118,6 +118,10 @@ where
|
||||
"WebDAV server starting"
|
||||
);
|
||||
|
||||
if self.config.tls_enabled && self.config.cert_dir.is_some() {
|
||||
rustfs_tls_runtime::ensure_crypto_provider().map_err(|err| WebDavInitError::Tls(err.to_string()))?;
|
||||
}
|
||||
|
||||
let listener = TcpListener::bind(self.config.bind_addr).await?;
|
||||
self.serve(listener, shutdown_rx).await
|
||||
}
|
||||
@@ -155,8 +159,6 @@ where
|
||||
reload_shutdown_rx.clone(),
|
||||
);
|
||||
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
let server_config = ServerConfig::builder().with_no_client_auth().with_cert_resolver(resolver);
|
||||
|
||||
Some(TlsAcceptor::from(Arc::new(server_config)))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1092,56 +1092,6 @@ impl_canonical_mutation_body!(
|
||||
// its JSON compatibility copy — so tampering with either encoding, or stripping `_bin` to force
|
||||
// the JSON fallback decode path, breaks the signature-bound digest.
|
||||
|
||||
pub fn canonical_metadata_reservation_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-request-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.object)?;
|
||||
body.push_u64(request.timeout_ms);
|
||||
body.push_bytes(&request.bucket_incarnation_id)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_metadata_reservation_cancel_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationCancelRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-cancel-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_metadata_reservation_activate_request_body(
|
||||
request: &proto_gen::node_service::MetadataReservationActivateRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
|
||||
body.push_str(&request.disk)?;
|
||||
body.push_str(&request.bucket)?;
|
||||
body.push_str(&request.object)?;
|
||||
body.push_bytes(&request.token)?;
|
||||
// Keep ordinary activations byte-compatible with existing peers. Renewal
|
||||
// adds a distinct signed marker so an older peer cannot treat it as activation.
|
||||
if request.renew_only {
|
||||
body.push_bool(true);
|
||||
}
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_reserved_rename_data_request_body(
|
||||
request: &proto_gen::node_service::ReservedRenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = CanonicalBodyBuilder::new(b"rustfs-reserved-rename-data-v1\0");
|
||||
if let Some(rename) = request.rename.as_ref() {
|
||||
body.push_bytes(&canonical_rename_data_request_body(rename)?)?;
|
||||
} else {
|
||||
body.push_bytes(&[])?;
|
||||
}
|
||||
body.push_bytes(&request.token)?;
|
||||
Ok(body.finish())
|
||||
}
|
||||
|
||||
pub fn canonical_rename_data_request_body(
|
||||
request: &proto_gen::node_service::RenameDataRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
@@ -1772,96 +1722,6 @@ mod disk_mutation_canonical_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_reservation_canonical_bodies_bind_scope_and_token() {
|
||||
use proto_gen::node_service::{
|
||||
MetadataReservationActivateRequest, MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
ReservedRenameDataRequest,
|
||||
};
|
||||
|
||||
let request = MetadataReservationRequest {
|
||||
disk: "disk".into(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
timeout_ms: 5000,
|
||||
bucket_incarnation_id: vec![1; 16].into(),
|
||||
};
|
||||
let mut bodies = vec![canonical_metadata_reservation_request_body(&request).expect("reservation body")];
|
||||
for mutate in [
|
||||
|r: &mut MetadataReservationRequest| r.disk = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.bucket = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.object = "other".into(),
|
||||
|r: &mut MetadataReservationRequest| r.timeout_ms = 1,
|
||||
|r: &mut MetadataReservationRequest| r.bucket_incarnation_id = vec![2; 16].into(),
|
||||
] {
|
||||
let mut changed = request.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_metadata_reservation_request_body(&changed).expect("changed body"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let activation = MetadataReservationActivateRequest {
|
||||
disk: "disk".into(),
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
token: vec![1; 16].into(),
|
||||
renew_only: false,
|
||||
};
|
||||
let mut legacy_activation = CanonicalBodyBuilder::new(b"rustfs-metadata-reservation-activate-v1\0");
|
||||
legacy_activation.push_str(&activation.disk).expect("disk");
|
||||
legacy_activation.push_str(&activation.bucket).expect("bucket");
|
||||
legacy_activation.push_str(&activation.object).expect("object");
|
||||
legacy_activation.push_bytes(&activation.token).expect("token");
|
||||
assert_eq!(
|
||||
canonical_metadata_reservation_activate_request_body(&activation).expect("ordinary activation"),
|
||||
legacy_activation.finish(),
|
||||
"ordinary activation must retain its previous body digest"
|
||||
);
|
||||
let mut bodies = vec![canonical_metadata_reservation_activate_request_body(&activation).expect("activation body")];
|
||||
for mutate in [
|
||||
|r: &mut MetadataReservationActivateRequest| r.disk = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.bucket = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.object = "other".into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.token = vec![2; 16].into(),
|
||||
|r: &mut MetadataReservationActivateRequest| r.renew_only = true,
|
||||
] {
|
||||
let mut changed = activation.clone();
|
||||
mutate(&mut changed);
|
||||
bodies.push(canonical_metadata_reservation_activate_request_body(&changed).expect("changed activation"));
|
||||
}
|
||||
assert_all_distinct(&bodies);
|
||||
let cancellation = MetadataReservationCancelRequest {
|
||||
disk: "disk".into(),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut changed = cancellation.clone();
|
||||
changed.token = vec![2; 16].into();
|
||||
assert_ne!(
|
||||
canonical_metadata_reservation_cancel_request_body(&cancellation).expect("cancel body"),
|
||||
canonical_metadata_reservation_cancel_request_body(&changed).expect("changed cancel body")
|
||||
);
|
||||
let rename = ReservedRenameDataRequest {
|
||||
rename: Some(RenameDataRequest {
|
||||
dst_path: "object".into(),
|
||||
..Default::default()
|
||||
}),
|
||||
token: vec![1; 16].into(),
|
||||
};
|
||||
let mut changed_token = rename.clone();
|
||||
changed_token.token = vec![2; 16].into();
|
||||
let mut changed_object = rename.clone();
|
||||
changed_object.rename.as_mut().expect("rename").dst_path = "other".into();
|
||||
assert_all_distinct(&[
|
||||
canonical_reserved_rename_data_request_body(&rename).expect("rename body"),
|
||||
canonical_reserved_rename_data_request_body(&changed_token).expect("changed token"),
|
||||
canonical_reserved_rename_data_request_body(&changed_object).expect("changed object"),
|
||||
canonical_reserved_rename_data_request_body(&ReservedRenameDataRequest {
|
||||
rename: None,
|
||||
token: rename.token,
|
||||
})
|
||||
.expect("missing rename"),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_data_canonical_body_binds_every_field() {
|
||||
let baseline = RenameDataRequest {
|
||||
|
||||
@@ -339,32 +339,6 @@ message RenameDataResponse {
|
||||
bytes rename_data_resp_bin = 4;
|
||||
}
|
||||
|
||||
message MetadataReservationRequest {
|
||||
string disk = 1;
|
||||
string bucket = 2;
|
||||
string object = 3;
|
||||
uint64 timeout_ms = 4;
|
||||
bytes bucket_incarnation_id = 5;
|
||||
}
|
||||
|
||||
message MetadataReservationCancelRequest {
|
||||
string disk = 1;
|
||||
bytes token = 2;
|
||||
}
|
||||
|
||||
message MetadataReservationActivateRequest {
|
||||
string disk = 1;
|
||||
string bucket = 2;
|
||||
string object = 3;
|
||||
bytes token = 4;
|
||||
bool renew_only = 5;
|
||||
}
|
||||
|
||||
message ReservedRenameDataRequest {
|
||||
RenameDataRequest rename = 1;
|
||||
bytes token = 2;
|
||||
}
|
||||
|
||||
// Fixed-scope native pool.bin recovery. The typed MessagePack payload never
|
||||
// accepts an arbitrary volume or object path.
|
||||
message PoolMetaNativeRecoveryRequest {
|
||||
@@ -1340,10 +1314,6 @@ service NodeService {
|
||||
rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; // auth-policy: read-only
|
||||
rpc WalkDir(WalkDirRequest) returns (stream WalkDirResponse) {}; // auth-policy: streaming
|
||||
rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc ReserveObjectMetadata(MetadataReservationRequest) returns (SnapshotLeaseResponse) {}; // auth-policy: body-bound
|
||||
rpc ActivateObjectMetadataReservation(MetadataReservationActivateRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
|
||||
rpc CancelObjectMetadataReservation(MetadataReservationCancelRequest) returns (SnapshotLeaseMutationResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameReservedData(ReservedRenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
// The method itself requires source-preserving multipart publication on every peer.
|
||||
rpc RenameMultipartData(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
rpc RenameDataAtIncarnation(RenameDataRequest) returns (RenameDataResponse) {}; // auth-policy: body-bound
|
||||
|
||||
@@ -1081,7 +1081,7 @@ impl ChecksumHasher for Sha512Hasher {
|
||||
/// MD5 hasher for the ADDITIONAL checksum (x-amz-checksum-md5). Separate from the
|
||||
/// legacy Content-MD5 / ETag machinery — this only serves the flexible-checksum path.
|
||||
pub struct Md5Hasher {
|
||||
hasher: md5::Md5,
|
||||
hasher: rustfs_utils::FastMd5,
|
||||
}
|
||||
|
||||
impl Default for Md5Hasher {
|
||||
@@ -1092,14 +1092,14 @@ impl Default for Md5Hasher {
|
||||
|
||||
impl Md5Hasher {
|
||||
pub fn new() -> Self {
|
||||
use md5::Digest as _;
|
||||
Self { hasher: md5::Md5::new() }
|
||||
Self {
|
||||
hasher: rustfs_utils::FastMd5::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Md5Hasher {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
use md5::Digest as _;
|
||||
self.hasher.update(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
@@ -1111,13 +1111,11 @@ impl Write for Md5Hasher {
|
||||
|
||||
impl ChecksumHasher for Md5Hasher {
|
||||
fn finalize(&mut self) -> Vec<u8> {
|
||||
use md5::Digest as _;
|
||||
self.hasher.clone().finalize().to_vec()
|
||||
self.hasher.finalize_bytes().to_vec()
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
use md5::Digest as _;
|
||||
self.hasher = md5::Md5::new();
|
||||
self.hasher.reset();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+102
-28
@@ -15,8 +15,8 @@
|
||||
use crate::compress_index::{Index, TryGetIndex};
|
||||
use crate::{BadDigest, EtagResolvable, HashReaderDetector, HashReaderMut};
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use md5::{Digest, Md5};
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_utils::FastMd5;
|
||||
use sha2::Sha256;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
@@ -27,11 +27,12 @@ pin_project! {
|
||||
pub struct EtagReader<R> {
|
||||
#[pin]
|
||||
pub inner: R,
|
||||
pub md5: Md5,
|
||||
pub md5: FastMd5,
|
||||
pub finished: bool,
|
||||
pub checksum: Option<String>,
|
||||
resolved_etag: Option<String>,
|
||||
encrypted_etag_mac: Option<Hmac<Sha256>>,
|
||||
delegate_inner_etag: bool,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,11 +40,12 @@ impl<R> EtagReader<R> {
|
||||
pub fn new(inner: R, checksum: Option<String>) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
md5: Md5::new(),
|
||||
md5: FastMd5::new(),
|
||||
finished: false,
|
||||
checksum,
|
||||
resolved_etag: None,
|
||||
encrypted_etag_mac: None,
|
||||
delegate_inner_etag: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +60,17 @@ impl<R> EtagReader<R> {
|
||||
Ok(reader)
|
||||
}
|
||||
|
||||
/// Reuse the inner reader's plaintext ETag when deriving an encrypted ETag,
|
||||
/// avoiding a redundant second MD5 pass over the same plaintext stream.
|
||||
pub fn with_inner_etag_delegation(mut self, delegate: bool) -> Self
|
||||
where
|
||||
R: EtagResolvable,
|
||||
{
|
||||
self.delegate_inner_etag =
|
||||
delegate && self.encrypted_etag_mac.is_some() && self.checksum.is_none() && self.inner.is_etag_reader();
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the final md5 value (etag) as a hex string, only compute once.
|
||||
/// Can be called multiple times, always returns the same result after finished.
|
||||
pub fn get_etag(&mut self) -> String {
|
||||
@@ -65,8 +78,8 @@ impl<R> EtagReader<R> {
|
||||
return etag.clone();
|
||||
}
|
||||
|
||||
let etag = self.md5.clone().finalize().to_vec();
|
||||
let etag = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
let digest = self.md5.finalize_bytes();
|
||||
let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
|
||||
self.resolved_etag = Some(etag.clone());
|
||||
etag
|
||||
}
|
||||
@@ -87,30 +100,32 @@ where
|
||||
if let Poll::Ready(Ok(())) = &poll {
|
||||
let filled = &buf.filled()[orig_filled..];
|
||||
if !filled.is_empty() {
|
||||
this.md5.update(filled);
|
||||
if !*this.delegate_inner_etag {
|
||||
this.md5.update(filled);
|
||||
}
|
||||
} else {
|
||||
// EOF
|
||||
*this.finished = true;
|
||||
let etag = if let Some(etag) = this.resolved_etag.as_ref() {
|
||||
etag.clone()
|
||||
} else {
|
||||
let etag = this.md5.clone().finalize().to_vec();
|
||||
let etag = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
*this.resolved_etag = Some(etag.clone());
|
||||
etag
|
||||
};
|
||||
if let Some(checksum) = this.checksum {
|
||||
let etag = if let Some(etag) = this.resolved_etag.as_ref() {
|
||||
etag.clone()
|
||||
} else {
|
||||
let digest = this.md5.finalize_bytes();
|
||||
let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower);
|
||||
*this.resolved_etag = Some(etag.clone());
|
||||
etag
|
||||
};
|
||||
|
||||
if let Some(checksum) = this.checksum
|
||||
&& *checksum != etag
|
||||
{
|
||||
error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
BadDigest {
|
||||
expected_md5: checksum.clone(),
|
||||
calculated_md5: etag,
|
||||
},
|
||||
)));
|
||||
if *checksum != etag {
|
||||
error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag);
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
BadDigest {
|
||||
expected_md5: checksum.clone(),
|
||||
calculated_md5: etag,
|
||||
},
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -118,16 +133,24 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> EtagResolvable for EtagReader<R> {
|
||||
impl<R> EtagResolvable for EtagReader<R>
|
||||
where
|
||||
R: EtagResolvable,
|
||||
{
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
true
|
||||
!self.delegate_inner_etag || self.inner.is_etag_reader()
|
||||
}
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
if let Some(mut mac) = self.encrypted_etag_mac.clone() {
|
||||
if !self.finished {
|
||||
return None;
|
||||
}
|
||||
mac.update(self.get_etag().as_bytes());
|
||||
let plaintext_etag = if self.delegate_inner_etag {
|
||||
self.inner.try_resolve_etag()?
|
||||
} else {
|
||||
self.get_etag()
|
||||
};
|
||||
mac.update(plaintext_etag.as_bytes());
|
||||
let protected = mac.finalize().into_bytes();
|
||||
return Some(hex_simd::encode_to_string(&protected[..16], hex_simd::AsciiCase::Lower));
|
||||
}
|
||||
@@ -167,6 +190,7 @@ where
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use md5::{Digest, Md5};
|
||||
use rand::RngExt;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::{AsyncReadExt, BufReader};
|
||||
@@ -340,4 +364,54 @@ mod tests {
|
||||
assert_eq!(digest.expected_md5, wrong_checksum);
|
||||
assert_eq!(digest.calculated_md5, calculated_md5);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn encrypted_etag_reader_with_inner_delegation_matches_standalone() {
|
||||
let data = b"delegated encrypted etag payload across multiple 64-byte MD5 blocks for verification";
|
||||
let key = [0x42; 32];
|
||||
let nonce = [0x24; 12];
|
||||
|
||||
let mut standalone =
|
||||
EtagReader::new_encrypted(Cursor::new(data.as_slice()), key, nonce).expect("standalone encrypted EtagReader");
|
||||
let mut out_standalone = Vec::new();
|
||||
standalone.read_to_end(&mut out_standalone).await.expect("read standalone");
|
||||
let expected_tag = standalone.try_resolve_etag().expect("standalone etag");
|
||||
|
||||
let inner =
|
||||
crate::HashReader::from_stream(Cursor::new(data.as_slice()), data.len() as i64, data.len() as i64, None, None, false)
|
||||
.expect("inner HashReader");
|
||||
assert!(inner.is_etag_reader(), "HashReader with diskable_md5=false must report is_etag_reader");
|
||||
let mut delegated = EtagReader::new_encrypted(inner, key, nonce)
|
||||
.expect("delegated encrypted EtagReader")
|
||||
.with_inner_etag_delegation(true);
|
||||
let mut out_delegated = Vec::new();
|
||||
delegated.read_to_end(&mut out_delegated).await.expect("read delegated");
|
||||
assert_eq!(out_delegated, data);
|
||||
assert_eq!(
|
||||
delegated.try_resolve_etag().as_deref(),
|
||||
Some(expected_tag.as_str()),
|
||||
"delegated encrypted EtagReader must produce identical HMAC ETag without re-hashing plaintext"
|
||||
);
|
||||
|
||||
let truncated = crate::LimitReader::new(
|
||||
crate::HashReader::from_stream(Cursor::new(data.as_slice()), data.len() as i64, data.len() as i64, None, None, false)
|
||||
.expect("inner HashReader for limit"),
|
||||
16,
|
||||
);
|
||||
assert!(
|
||||
!truncated.is_etag_reader(),
|
||||
"truncating LimitReader must not advertise inner ETag delegation"
|
||||
);
|
||||
|
||||
let mut non_etag_inner = EtagReader::new_encrypted(Cursor::new(data.as_slice()), key, nonce)
|
||||
.expect("encrypted EtagReader over Cursor")
|
||||
.with_inner_etag_delegation(true);
|
||||
let mut out_cursor = Vec::new();
|
||||
non_etag_inner.read_to_end(&mut out_cursor).await.expect("read cursor");
|
||||
assert_eq!(
|
||||
non_etag_inner.try_resolve_etag().as_deref(),
|
||||
Some(expected_tag.as_str()),
|
||||
"delegation request on non-EtagResolvable inner must keep outer MD5 hashing active"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -630,6 +630,10 @@ impl AsyncRead for HashReader {
|
||||
}
|
||||
|
||||
impl EtagResolvable for HashReader {
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
!self.diskable_md5 && self.inner.is_etag_reader()
|
||||
}
|
||||
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
if self.diskable_md5 {
|
||||
return None;
|
||||
|
||||
@@ -201,10 +201,31 @@ where
|
||||
boxed_reader(WarpReader::new(reader))
|
||||
}
|
||||
|
||||
impl<T> EtagResolvable for std::io::Cursor<T> {}
|
||||
|
||||
impl EtagResolvable for &[u8] {}
|
||||
|
||||
impl<R> EtagResolvable for tokio::io::BufReader<R>
|
||||
where
|
||||
R: tokio::io::AsyncRead + EtagResolvable,
|
||||
{
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
self.get_ref().is_etag_reader()
|
||||
}
|
||||
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
self.get_mut().try_resolve_etag()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> EtagResolvable for Box<T>
|
||||
where
|
||||
T: EtagResolvable + ?Sized,
|
||||
{
|
||||
fn is_etag_reader(&self) -> bool {
|
||||
self.as_ref().is_etag_reader()
|
||||
}
|
||||
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
self.as_mut().try_resolve_etag()
|
||||
}
|
||||
|
||||
@@ -25,6 +25,10 @@ keywords = ["s3", "client", "tiering", "rustfs", "Minio"]
|
||||
categories = ["web-programming", "development-tools", "network-programming"]
|
||||
documentation = "https://docs.rs/rustfs-s3-client/latest/rustfs_s3_client/"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-tls-runtime/fips"]
|
||||
|
||||
[dependencies]
|
||||
rustfs-checksums.workspace = true
|
||||
rustfs-config.workspace = true
|
||||
|
||||
@@ -81,7 +81,7 @@ use time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tracing::{debug, error, trace, warn};
|
||||
use tracing::{error, trace, warn};
|
||||
use url::{Url, form_urlencoded};
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -448,16 +448,7 @@ impl TransitionClient {
|
||||
tier_type: &str,
|
||||
timeouts: TransitionClientTimeouts,
|
||||
) -> Result<TransitionClient, std::io::Error> {
|
||||
if rustls::crypto::CryptoProvider::get_default().is_none() {
|
||||
// No default provider is set yet; try to install aws-lc-rs.
|
||||
// `install_default` can only fail if another thread races us and installs a provider
|
||||
// between our check and this call, which is still safe to ignore.
|
||||
if rustls::crypto::aws_lc_rs::default_provider().install_default().is_err() {
|
||||
debug!("rustls crypto provider was installed concurrently, skipping aws-lc-rs install");
|
||||
}
|
||||
} else {
|
||||
debug!("rustls crypto provider already installed, skipping aws-lc-rs install");
|
||||
}
|
||||
rustfs_tls_runtime::ensure_crypto_provider().map_err(std::io::Error::other)?;
|
||||
|
||||
let endpoint_url = get_endpoint_url(endpoint, opts.secure)?;
|
||||
let timeouts = timeouts.validate()?;
|
||||
@@ -1862,21 +1853,36 @@ mod tests {
|
||||
assert!(outcome.is_ok(), "TLS config creation should not panic");
|
||||
}
|
||||
|
||||
/// Installing the rustls crypto provider when one is already set must not panic or return
|
||||
/// an error that surfaces to callers (the race-safe `get_default` check guards the install).
|
||||
#[cfg(feature = "fips")]
|
||||
#[test]
|
||||
fn provider_install_is_idempotent() {
|
||||
// Install once (may already be set by another test in this binary — that's fine).
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
|
||||
// A second install attempt on an already-set provider must not panic.
|
||||
let outcome = std::panic::catch_unwind(|| {
|
||||
if rustls::crypto::CryptoProvider::get_default().is_none() {
|
||||
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
|
||||
}
|
||||
// If a default is already present, the branch above is simply skipped.
|
||||
});
|
||||
assert!(outcome.is_ok(), "provider install guard must not panic when a provider is already set");
|
||||
fn fips_transition_rejects_incompatible_host_provider() {
|
||||
const CHILD: &str = "RUSTFS_TEST_TRANSITION_FIPS_PROVIDER";
|
||||
if std::env::var_os(CHILD).is_some() {
|
||||
let mut provider = rustfs_tls_runtime::default_crypto_provider();
|
||||
provider.kx_groups = vec![rustls::crypto::aws_lc_rs::kx_group::X25519];
|
||||
provider.install_default().expect("install incompatible host provider");
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("test runtime");
|
||||
let error = runtime
|
||||
.block_on(super::TransitionClient::new("127.0.0.1:0", super::Options::default(), ""))
|
||||
.expect_err("transition client must reject the incompatible provider");
|
||||
assert!(error.to_string().contains("process-wide rustls crypto provider"));
|
||||
return;
|
||||
}
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.arg("fips_transition_rejects_incompatible_host_provider")
|
||||
.arg("--nocapture")
|
||||
.env(CHILD, "1")
|
||||
.output()
|
||||
.expect("start isolated transition TLS test");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"stdout={} stderr={}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -640,12 +640,12 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
|
||||
GetRange::Bounded(range) => http_range_spec_from_range(range.clone()),
|
||||
GetRange::Offset(start) => HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: *start as i64,
|
||||
start: i64::try_from(*start).unwrap_or(i64::MAX),
|
||||
end: -1,
|
||||
},
|
||||
GetRange::Suffix(length) => HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: *length as i64,
|
||||
start: i64::try_from(*length).unwrap_or(i64::MAX),
|
||||
end: -1,
|
||||
},
|
||||
}
|
||||
@@ -654,15 +654,15 @@ fn http_range_spec_from_get_range(range: &GetRange) -> HTTPRangeSpec {
|
||||
fn http_range_spec_from_range(range: Range<u64>) -> HTTPRangeSpec {
|
||||
HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: range.start as i64,
|
||||
end: range.end.saturating_sub(1) as i64,
|
||||
start: i64::try_from(range.start).unwrap_or(i64::MAX),
|
||||
end: i64::try_from(range.end.saturating_sub(1)).unwrap_or(i64::MAX),
|
||||
}
|
||||
}
|
||||
|
||||
fn http_range_spec_from_start(start: u64) -> HTTPRangeSpec {
|
||||
HTTPRangeSpec {
|
||||
is_suffix_length: false,
|
||||
start: start as i64,
|
||||
start: i64::try_from(start).unwrap_or(i64::MAX),
|
||||
end: -1,
|
||||
}
|
||||
}
|
||||
@@ -2476,6 +2476,48 @@ mod test {
|
||||
assert_eq!(range.end, 19);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_range_conversion_for_suffix_ranges() {
|
||||
// Normal suffix: last 100 bytes.
|
||||
let range = http_range_spec_from_get_range(&GetRange::Suffix(100));
|
||||
assert!(range.is_suffix_length);
|
||||
assert_eq!(range.start, 100);
|
||||
assert_eq!(range.end, -1);
|
||||
|
||||
// Overflowing suffix: `u64::MAX` previously wrapped to `-1` via `as i64`
|
||||
// and was misread as "last 1 byte". It now clamps to `i64::MAX`, which
|
||||
// `get_length` treats as "the whole object".
|
||||
let range = http_range_spec_from_get_range(&GetRange::Suffix(u64::MAX));
|
||||
assert!(range.is_suffix_length);
|
||||
assert_eq!(range.start, i64::MAX);
|
||||
assert_eq!(range.end, -1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_range_conversion_for_offset_overflow() {
|
||||
// Normal offset.
|
||||
let range = http_range_spec_from_get_range(&GetRange::Offset(42));
|
||||
assert!(!range.is_suffix_length);
|
||||
assert_eq!(range.start, 42);
|
||||
assert_eq!(range.end, -1);
|
||||
|
||||
// Overflowing offset clamps to `i64::MAX`, which is then rejected as
|
||||
// out-of-range (416) by `get_offset_length`.
|
||||
let range = http_range_spec_from_get_range(&GetRange::Offset(u64::MAX));
|
||||
assert!(!range.is_suffix_length);
|
||||
assert_eq!(range.start, i64::MAX);
|
||||
assert_eq!(range.end, -1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_range_conversion_for_bounded_overflow() {
|
||||
// Overflowing bounded range clamps start/end to `i64::MAX`.
|
||||
let range = http_range_spec_from_get_range(&GetRange::Bounded(u64::MAX - 10..u64::MAX));
|
||||
assert!(!range.is_suffix_length);
|
||||
assert_eq!(range.start, i64::MAX);
|
||||
assert_eq!(range.end, i64::MAX);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_scan_range_output_can_convert_field_delimiter() {
|
||||
let chunks = stream::iter(vec![Ok::<_, std::io::Error>(Bytes::from_static(b"a&&1\nb&&2\n"))]);
|
||||
|
||||
@@ -90,7 +90,9 @@ pub use bucket::BUCKET_LIFECYCLE_LOCK_OBJECT;
|
||||
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
|
||||
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::MultipartUploadInspection;
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use object::AuthoritativeListOperations;
|
||||
pub use object::DeleteAccounting;
|
||||
pub use object::ObjectLockDeleteOptions;
|
||||
pub use object::ObjectToDeleteIdentity;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user