mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-06 18:52:15 +00:00
Compare commits
63 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b61a127464 | |||
| d997f01d8d | |||
| ee173d469f | |||
| 7b499bbbd8 | |||
| 5b134dbe2d | |||
| 29fe1c6d46 | |||
| 6b70584dde | |||
| d6210ac17b | |||
| 158aa276e9 | |||
| 3362421240 | |||
| 57df9181d0 | |||
| 371f5e4ffc | |||
| 8a5a2c9c93 | |||
| babfeaed00 | |||
| 736bf777b4 | |||
| 7babc7d621 | |||
| 4d46bd25f8 | |||
| ff5f491c5a | |||
| a8be876fd9 | |||
| fe3ada0689 | |||
| 926355ae1f | |||
| 7f5fa4b8f6 | |||
| a97c65895c | |||
| 73e7afa01a | |||
| e0f26a540d | |||
| 3b3cc018aa | |||
| f4f5635aab | |||
| 36ad5a6b87 | |||
| 6c14128072 | |||
| addc258a24 | |||
| 73ea527f55 | |||
| 55da14ad82 | |||
| 116a5299b2 | |||
| fbc54f7212 | |||
| 3aa492f5ee | |||
| 00a81b4048 | |||
| 20ef9ebf15 | |||
| cc08e0d11b | |||
| 69294523d5 | |||
| cd4d86f28a | |||
| 2f55cf69e1 | |||
| fff6e94378 | |||
| 41b13aeb8b | |||
| 88587a8f94 | |||
| 8c8b5da63f | |||
| 00a238f838 | |||
| 965d805a4f | |||
| 9d66e04ae3 | |||
| 12c58d25b5 | |||
| 1ac1d883f4 | |||
| 45ba33291a | |||
| 636eded277 | |||
| ed4099ebe1 | |||
| 3a1fac16e3 | |||
| b00e85f1a4 | |||
| ecbddc188a | |||
| 2d63931bba | |||
| d429c6f792 | |||
| e494dc2b4b | |||
| 2b7c927ee8 | |||
| 647846956f | |||
| 04690f7da5 | |||
| 3c0829049d |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
|
||||
sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
|
||||
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
|
||||
sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
|
||||
sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=3814cdde8cacf4187e18188d342ed70931cc32e268e42e5bb438d565bb4d0f81
|
||||
sha256-linux=663025a80273579f95ae3f49af02c0afcbfce94df09701c360a13e607b7637db
|
||||
sha256-darwin=e84e04208f19dc0596b10d4d1b799ce43d6cc431415902f9c5458aa8232bc2d8
|
||||
sha256-linux=1f9c7add72c44d1c12ef4ac050bb7917431be657c0c90b29f70c3df56f8bbae3
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
|
||||
sha256=5bdd4c2eadbc41625ae3184fff34062e000d328108436259b2b7c4fe2531ecaf
|
||||
|
||||
+13
-7
@@ -234,7 +234,7 @@ test-group = 'embedded-test-ports'
|
||||
# processes cannot exhaust those budgets. Keep deadlines and assertions intact.
|
||||
# The short HTTP fixtures are tracked in rustfs/backlog#2714.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# This cancellation probe hashes the serving executable before acquiring its
|
||||
@@ -548,7 +548,7 @@ filter = 'package(rustfs) & (binary(/^embedded.*_test$/) | binary(admin_diagnost
|
||||
test-group = 'embedded-test-ports'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
filter = 'package(rustfs) & (binary(/^connect_registration$/) | binary(connect_inventory) | binary(connect_perf_drive) | (binary(connect_perf_client) & test(=configured_proxy_and_direct_endpoint_failures_are_distinct)) | (binary(connect_perf_object) & test(/^(real_rustfs_endpoint_and_production_cli_support_bounded_get_and_put|cleanup_uses_a_budget_independent_of_the_measurement_window|cleanup_accepts_missing_object_and_still_deletes_the_bucket|cleanup_diagnostics_identify_each_disconnected_transport_stage|cleanup_diagnostics_identify_each_failed_http_stage_without_private_context|cleanup_diagnostics_identify_timeouts_without_resetting_the_shared_deadline|oversized_response_is_rejected_without_buffering_it)$/)))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Preserve the cancellation probe's startup budget and assertions in CI too.
|
||||
@@ -701,7 +701,7 @@ test-group = 'ecstore-serial-flaky'
|
||||
[profile.e2e-smoke]
|
||||
default-filter = """
|
||||
package(e2e_test) & (
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
|
||||
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
|
||||
| test(/^reliant::lifecycle::/)
|
||||
| test(/^conditional_delete_objects_test::/)
|
||||
@@ -827,7 +827,7 @@ slow-timeout = { period = "120s", terminate-after = 12, grace-period = "10s" }
|
||||
# ---------------------------------------------------------------------------
|
||||
# e2e-distributed profile — 4-node 4-disk Actions suite
|
||||
# ---------------------------------------------------------------------------
|
||||
# Storage-sensitive PR / nightly / dispatch lane owned by
|
||||
# Nightly and manual-dispatch lane owned by
|
||||
# .github/workflows/e2e-distributed.yml.
|
||||
# Each case starts four rustfs processes (and for site replication, two
|
||||
# clusters). Upgrade cases also require RUSTFS_UPGRADE_SOURCE_BINARY.
|
||||
@@ -970,10 +970,16 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
# Keep storage-heavy cases isolated in the full suite too. The large pagination
|
||||
# fixture alone seeds 2,000 objects and can otherwise starve other servers.
|
||||
# Keep the storage-heavy smoke cases isolated in the full suite too. They
|
||||
# start multiple server fixtures and issue many sequential S3 operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter|test_list_objects_v2_raw_exceeds_maxkeys_but_visible_below)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
filter = 'package(e2e_test) & (test(/^list_objects_v2_pagination_test::tests::(test_list_objects_v2_delimiter_small_page_traverses_all|test_list_objects_v2_max_keys_above_limit_returns_token|test_list_objects_v2_maxkeys_above_limit_with_delimiter)$/) | test(=reliant::tiering::test_hermetic_transition_restore_failure_expiry_and_retry))'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# The mc fixture uploads hundreds of objects concurrently before measuring listing.
|
||||
# Reserve the runner so other server fixtures do not starve its disk operations.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=mc_mirror_small_bucket_test::test_mc_mirror_small_bucket_completes_without_list_timeout)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
# Fixed, small contract for every executable server change. These existing
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ The following files in the project root are essential for Docker operations:
|
||||
| File | Description | Usage |
|
||||
| :--- | :--- | :--- |
|
||||
| **`docker-buildx.sh`** | **Multi-Arch Build Script**<br>Automates building and pushing Docker images for `amd64` and `arm64`. Supports release and dev channels. | `./docker-buildx.sh --push` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads pre-built binaries from GitHub Releases. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Useful if you need specific dynamic libraries. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code. Includes build tools. Ideal for local development and CI. | `docker build -f Dockerfile.source .` |
|
||||
| **`Dockerfile`** | **Production Image (Alpine)**<br>Lightweight image using musl libc. Downloads verified RustFS and `rc` binaries from GitHub Releases and bundles `/etc/rustfs/bootstrap`. | `docker build -t rustfs:latest .` |
|
||||
| **`Dockerfile.glibc`** | **Production Image (Ubuntu)**<br>Standard image using glibc. Includes the bundled bootstrap configuration. | `docker build -f Dockerfile.glibc .` |
|
||||
| **`Dockerfile.source`** | **Development Image**<br>Builds RustFS from source code and bundles `rc` and the default bootstrap configuration. | `docker build -f Dockerfile.source .` |
|
||||
|
||||
### Docker Compose Configurations
|
||||
|
||||
|
||||
@@ -32,6 +32,8 @@ updates:
|
||||
- "overtrue"
|
||||
- "majinghe"
|
||||
ignore:
|
||||
- dependency-name: "pulsar"
|
||||
versions: [ "6.9.x" ]
|
||||
- dependency-name: "object_store"
|
||||
versions: [ "0.13.x" ]
|
||||
- dependency-name: "libunftp"
|
||||
|
||||
@@ -296,6 +296,32 @@
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/rustfs-validation-sweep.yml",
|
||||
"name": "RustFS Validation Sweep",
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"30 19 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 60,
|
||||
"result_owner": "rustfs-validation-sweep.yml",
|
||||
"disposition": "canonical validation cases executed in rotating batches",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/btf-prototype-reference.yml",
|
||||
"name": "BTF prototype reference",
|
||||
@@ -743,12 +769,10 @@
|
||||
"workflow": ".github/workflows/e2e-distributed.yml",
|
||||
"name": "e2e-distributed",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"workflow_dispatch",
|
||||
"schedule"
|
||||
],
|
||||
@@ -823,12 +847,11 @@
|
||||
"workflow": ".github/workflows/e2e-upgrade.yml",
|
||||
"name": "Upgrade Compatibility",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"push",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
|
||||
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -497,6 +497,63 @@ jobs:
|
||||
- name: Check offline enrollment E2E root boundary
|
||||
run: ./scripts/check_offline_enrollment_e2e.sh
|
||||
|
||||
fips-provider:
|
||||
name: FIPS provider boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# This builds the RustFS library test target, which exceeds hosted-runner disk capacity.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [default, fips]
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CARGO_BUILD_JOBS: "2"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-fips-provider
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Install FIPS build tools
|
||||
if: matrix.mode == 'fips'
|
||||
run: sudo apt-get install -y cmake golang clang libclang-dev
|
||||
|
||||
- name: Verify provider installation
|
||||
shell: bash
|
||||
run: |
|
||||
args=()
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
args+=(--features fips)
|
||||
fi
|
||||
cargo nextest run --locked -p rustfs-tls-runtime --lib "${args[@]}" --no-tests fail -E 'test(provider_policy_isolated)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib "${args[@]}" --no-tests fail -E 'test(overflowing_closed_range_is_rejected_before_network_io)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(validate_pulsar)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib "${args[@]}" --no-tests fail -E 'test(webhook_provider_policy_isolated)'
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
cargo nextest run --locked -p rustfs-protocols --lib --features ftps,webdav,fips --no-tests fail -E 'test(protocols_reject_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-targets --lib --features fips --no-tests fail -E 'test(fips_postgres_rejects_incompatible_host_provider)'
|
||||
cargo nextest run --locked -p rustfs-s3-client --lib --features fips --no-tests fail -E 'test(fips_transition_rejects_incompatible_host_provider)'
|
||||
fi
|
||||
tests=(provider_installation_isolated prepared_generation_shares_crypto_and_serves_sni_with_mtls)
|
||||
if [[ '${{ matrix.mode }}' == 'fips' ]]; then
|
||||
tests+=(fips_provider_builds_fips_tls_configs fips_provider_tls_handshake_accepts_and_rejects_algorithms)
|
||||
fi
|
||||
for test_name in "${tests[@]}"; do
|
||||
cargo nextest run --locked -p rustfs --lib "${args[@]}" --no-tests fail -E "test($test_name)"
|
||||
done
|
||||
cargo clippy --locked -p rustfs --lib "${args[@]}" -- -D warnings
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
@@ -1397,6 +1454,7 @@ jobs:
|
||||
- workspace-shard-evidence
|
||||
- core-contract
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- test-and-lint-rio-v2
|
||||
- test-and-lint-protocols
|
||||
- build-rustfs-debug-binary
|
||||
@@ -1424,6 +1482,7 @@ jobs:
|
||||
needs:
|
||||
- classify-changes
|
||||
- connect-feature-boundaries
|
||||
- fips-provider
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Run CLA Bot
|
||||
if: github.event_name != 'merge_group'
|
||||
uses: overtrue/cla-bot@5e5929846a60cfb42b16d0d38e135ef67d94fc88 # v0.0.9
|
||||
uses: overtrue/cla-bot@f054da99d5777e47e1af9933737b9c0c5584be8d # v0.1.0
|
||||
with:
|
||||
github-token: ${{ github.token }}
|
||||
registry-token: ${{ steps.registry-token.outputs.token }}
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
# Each selected test starts a real localhost cluster via
|
||||
# `RustFSTestClusterEnvironment` (4 processes; 4 drives per node unless the
|
||||
# case is a two-site 4-node 1-drive pair or a 4-node upgrade). Membership is
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. Storage-sensitive PRs,
|
||||
# nightly runs, and manual dispatches all execute the same fail-closed suite.
|
||||
# `[profile.e2e-distributed]` in `.config/nextest.toml`. The nightly schedule
|
||||
# and manual dispatches execute the same fail-closed suite.
|
||||
# Upgrade cases download the same pinned previous release as e2e-upgrade.yml.
|
||||
#
|
||||
# Isolated pool filesystems: expand/decommission/rebalance cases require
|
||||
@@ -33,32 +33,6 @@
|
||||
name: e2e-distributed
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- "Cargo.lock"
|
||||
- "Cargo.toml"
|
||||
- ".config/nextest.toml"
|
||||
- ".config/e2e-distributed-selection.txt"
|
||||
- ".github/workflows/e2e-distributed.yml"
|
||||
- "crates/audit/**"
|
||||
- "crates/common/**"
|
||||
- "crates/config/**"
|
||||
- "crates/e2e_test/**"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/iam/**"
|
||||
- "crates/lock/**"
|
||||
- "crates/madmin/**"
|
||||
- "crates/notify/**"
|
||||
- "crates/replication/**"
|
||||
- "crates/s3-client/**"
|
||||
- "crates/s3-ops/**"
|
||||
- "crates/s3-types/**"
|
||||
- "crates/scanner/**"
|
||||
- "crates/storage-api/**"
|
||||
- "crates/utils/**"
|
||||
- "rustfs/**"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
|
||||
@@ -12,30 +12,19 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Single-node upgrade and rollback lane.
|
||||
#
|
||||
# The weekly Monday schedule, version tags, and workflow_dispatch run this
|
||||
# suite. Distributed upgrade cases stay on the nightly e2e-distributed lane.
|
||||
|
||||
name: Upgrade Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/e2e-upgrade.yml"
|
||||
- "crates/e2e_test/src/common.rs"
|
||||
- "crates/e2e_test/src/distributed/**"
|
||||
- "crates/e2e_test/src/fake_s3_target/**"
|
||||
- "crates/e2e_test/src/lib.rs"
|
||||
- "crates/e2e_test/src/replication_extension_test.rs"
|
||||
- "crates/e2e_test/src/rpc_fault_proxy.rs"
|
||||
- "crates/e2e_test/src/upgrade_compatibility_test.rs"
|
||||
- "crates/ecstore/**"
|
||||
- "crates/filemeta/**"
|
||||
- "crates/heal/**"
|
||||
- "crates/kms/**"
|
||||
- "crates/storage-api/**"
|
||||
- "rustfs/**"
|
||||
- "Cargo.lock"
|
||||
push:
|
||||
tags:
|
||||
- "[0-9]*.[0-9]*.[0-9]*"
|
||||
schedule:
|
||||
# 03:17 UTC every Monday.
|
||||
- cron: "17 3 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -44,7 +33,7 @@ permissions:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -87,18 +87,206 @@ jobs:
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Download static console assets
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
# Packaging tools are off on this lane, so setup does not install unzip.
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq unzip
|
||||
fi
|
||||
if ! command -v unzip >/dev/null 2>&1; then
|
||||
echo "unzip is required to extract console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p ./rustfs/static
|
||||
|
||||
verify_sha256() {
|
||||
local expected="$1"
|
||||
local file="$2"
|
||||
local actual=""
|
||||
local actual_line=""
|
||||
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
actual_line=$(sha256sum -- "$file") || return 1
|
||||
elif command -v shasum >/dev/null 2>&1; then
|
||||
actual_line=$(shasum -a 256 -- "$file") || return 1
|
||||
else
|
||||
echo "No SHA-256 verification tool found" >&2
|
||||
return 1
|
||||
fi
|
||||
actual="${actual_line%%[[:space:]]*}"
|
||||
|
||||
if [ "$actual" = "$expected" ]; then
|
||||
echo "SHA256 verified OK: $actual"
|
||||
return 0
|
||||
else
|
||||
echo "SHA256 mismatch: expected=$expected actual=$actual" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
download_console_assets() {
|
||||
local curl_bin="curl"
|
||||
local python_bin="python3"
|
||||
local console_api="https://api.github.com/repos/rustfs/console/releases/latest"
|
||||
local console_json="console-release.json"
|
||||
local console_url
|
||||
local console_sha256
|
||||
local curl_auth_args=()
|
||||
console_tag=""
|
||||
|
||||
chmod +w ./rustfs/static/LICENSE || true
|
||||
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
python_bin="python"
|
||||
fi
|
||||
if ! command -v "$python_bin" >/dev/null 2>&1; then
|
||||
echo "No Python interpreter found for release metadata parsing" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||||
curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28")
|
||||
fi
|
||||
|
||||
"$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \
|
||||
-o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
|
||||
read -r console_tag console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
release = json.load(handle)
|
||||
|
||||
tag = release.get("tag_name", "") or "unknown"
|
||||
|
||||
for asset in release.get("assets", []):
|
||||
name = asset.get("name", "")
|
||||
digest = asset.get("digest", "")
|
||||
url = asset.get("browser_download_url", "")
|
||||
if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url:
|
||||
sha256 = digest.split(":", 1)[1]
|
||||
if not re.fullmatch(r"[0-9a-fA-F]{64}", sha256):
|
||||
raise SystemExit(f"console zip asset has invalid sha256 digest: {sha256}")
|
||||
sys.stdout.buffer.write(f"{tag} {url} {sha256}\n".encode("utf-8"))
|
||||
break
|
||||
else:
|
||||
raise SystemExit("no console zip asset with sha256 digest found")
|
||||
PY
|
||||
) || return 1
|
||||
|
||||
if [[ -z "$console_url" || -z "$console_sha256" ]]; then
|
||||
echo "Console release metadata is missing URL or SHA-256 digest" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Console release: ${console_tag}"
|
||||
echo "Downloading console asset: ${console_url}"
|
||||
"$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1
|
||||
verify_sha256 "$console_sha256" console.zip || return 2
|
||||
unzip -o console.zip -d ./rustfs/static || return 2
|
||||
}
|
||||
|
||||
if download_console_assets; then
|
||||
rm -f console.zip console-release.json
|
||||
else
|
||||
status=$?
|
||||
rm -f console.zip console-release.json
|
||||
if [[ "$status" -eq 2 ]]; then
|
||||
echo "Console asset integrity verification failed" >&2
|
||||
fi
|
||||
echo "Failed to download verified console assets" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -s ./rustfs/static/index.html ]]; then
|
||||
echo "Console asset archive is missing static/index.html" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
asset_count=$(find ./rustfs/static -type f | wc -l | tr -d '[:space:]')
|
||||
echo "Console assets ready: version=${console_tag:-unknown}, ${asset_count} files extracted to ./rustfs/static"
|
||||
|
||||
- name: Build RustFS
|
||||
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
|
||||
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
|
||||
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
|
||||
shell: bash
|
||||
run: |
|
||||
# Force rebuild by touching build.rs so a restored cache cannot
|
||||
# reuse a rustfs artifact compiled before these console assets existed.
|
||||
touch rustfs/build.rs
|
||||
cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bin rustfs
|
||||
|
||||
- name: Verify glibc symbol baseline
|
||||
run: |
|
||||
python3 scripts/release/check_glibc.py 2.31 \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs-cli
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
|
||||
|
||||
- name: Verify console
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
find_free_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PY
|
||||
}
|
||||
|
||||
binary="target/x86_64-unknown-linux-gnu/release/rustfs"
|
||||
data_dir="$(mktemp -d)"
|
||||
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
||||
server_pid=""
|
||||
trap '
|
||||
if [[ -n "${server_pid:-}" ]]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$data_dir"
|
||||
' EXIT
|
||||
|
||||
if [[ ! -x "$binary" ]]; then
|
||||
echo "rustfs binary not found: ${binary}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
api_port="$(find_free_port)"
|
||||
console_port="$(find_free_port)"
|
||||
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
||||
|
||||
"$binary" server \
|
||||
--address "127.0.0.1:${api_port}" \
|
||||
--console-enable \
|
||||
--console-address "127.0.0.1:${console_port}" \
|
||||
--access-key console-smoke \
|
||||
--secret-key console-smoke-secret \
|
||||
"$data_dir" >"$log_file" 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
response=""
|
||||
for _ in {1..40}; do
|
||||
response="$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)"
|
||||
if [[ "$response" == 200\ text/html* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
echo "Console endpoint did not return 200 text/html: $response" >&2
|
||||
cat "$log_file"
|
||||
exit 1
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
shell: bash
|
||||
@@ -346,12 +534,12 @@ jobs:
|
||||
path: ${{ steps.publish.outputs.candidate_file }}
|
||||
if-no-files-found: error
|
||||
|
||||
# Publish the deb/rpm pair to the auto-testing repo's `assets` branch so
|
||||
# engineers can download and install the nightly directly. The branch is
|
||||
# a single-commit orphan rewritten on every build, which keeps the repo
|
||||
# small while the latest files stay reachable at stable raw URLs.
|
||||
# Publish the deb/rpm pair as assets of the rolling `nightly` release on
|
||||
# rustfs/auto-testing (see scripts/release/publish_nightly_assets.sh).
|
||||
# The dispatch below runs only after this upload and the immutable R2
|
||||
# candidate both succeed. It does not replace rustfs-functional-chain.yml,
|
||||
# which still starts from this workflow's workflow_run completion.
|
||||
# kms-vault-lane and kms-vault-ha-failover stay parallel with this job.
|
||||
- name: Publish packages to auto-testing release assets
|
||||
env:
|
||||
ASSETS_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -360,6 +548,90 @@ jobs:
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
BUILD_REF: ${{ env.NIGHTLY_BUILD_REF }}
|
||||
run: bash scripts/release/publish_nightly_assets.sh
|
||||
|
||||
# Cross-repo repository_dispatch cannot use GITHUB_TOKEN. The secret is a
|
||||
# PAT or GitHub App token for rustfs/auto-testing: classic `repo` scope,
|
||||
# or fine-grained Contents write (the permission GitHub checks for
|
||||
# POST /repos/{owner}/{repo}/dispatches). A dispatch failure fails this
|
||||
# step, and therefore this job, so the scheduled failure alert fires.
|
||||
# GitHub allows at most 10 top-level client_payload properties, so the
|
||||
# schema 2 candidate object is nested under `candidate` and the release
|
||||
# fields sit beside it.
|
||||
- name: Dispatch nightly candidate to auto-testing
|
||||
if: ${{ steps.publish.outputs.candidate_file != '' }}
|
||||
env:
|
||||
DISPATCH_TOKEN: ${{ secrets.AUTO_TESTING_DISPATCH_TOKEN }}
|
||||
CANDIDATE_FILE: ${{ steps.publish.outputs.candidate_file }}
|
||||
DEB_DATE: ${{ steps.deb.outputs.deb_date }}
|
||||
SOURCE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${DISPATCH_TOKEN:?AUTO_TESTING_DISPATCH_TOKEN is required to dispatch rustfs/auto-testing}"
|
||||
: "${CANDIDATE_FILE:?nightly candidate manifest is required}"
|
||||
: "${DEB_DATE:?DEB_DATE is required}"
|
||||
[[ "${DEB_DATE}" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]]
|
||||
[[ "${SOURCE_REPO}" == "rustfs/rustfs" ]]
|
||||
|
||||
payload="${RUNNER_TEMP}/nightly-candidate-dispatch-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile candidate "${CANDIDATE_FILE}" \
|
||||
--arg event_type "nightly-candidate-ready" \
|
||||
--arg source_repo "${SOURCE_REPO}" \
|
||||
--arg latest_alias_url "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb" \
|
||||
--arg auto_testing_release "https://github.com/rustfs/auto-testing/releases/tag/nightly" \
|
||||
--arg channel "nightly" \
|
||||
--arg deb_date "${DEB_DATE}" \
|
||||
--arg nightly_workflow ".github/workflows/nightly-gnu.yml" \
|
||||
'{
|
||||
event_type: $event_type,
|
||||
source_repo: $source_repo,
|
||||
latest_alias_url: $latest_alias_url,
|
||||
auto_testing_release: $auto_testing_release,
|
||||
channel: $channel,
|
||||
deb_date: $deb_date,
|
||||
nightly_workflow: $nightly_workflow,
|
||||
candidate: $candidate[0]
|
||||
}' > "${payload}"
|
||||
if ! jq -e '
|
||||
.event_type == "nightly-candidate-ready"
|
||||
and .source_repo == "rustfs/rustfs"
|
||||
and .channel == "nightly"
|
||||
and .deb_date == env.DEB_DATE
|
||||
and .nightly_workflow == ".github/workflows/nightly-gnu.yml"
|
||||
and .latest_alias_url == "https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb"
|
||||
and .auto_testing_release == "https://github.com/rustfs/auto-testing/releases/tag/nightly"
|
||||
and .candidate.schema == 2
|
||||
and ((.candidate | keys | sort) == ["build_run_attempt","build_run_id","package_sha256","package_url","schema","source_ref","source_sha","workflow_sha"])
|
||||
and ((keys | sort) == ["auto_testing_release","candidate","channel","deb_date","event_type","latest_alias_url","nightly_workflow","source_repo"])
|
||||
' "${payload}" >/dev/null; then
|
||||
echo "nightly dispatch payload does not match the nightly-candidate-ready contract" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
body="${RUNNER_TEMP}/nightly-candidate-dispatch-body-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json"
|
||||
jq -n --slurpfile client_payload "${payload}" \
|
||||
'{event_type: "nightly-candidate-ready", client_payload: $client_payload[0]}' > "${body}"
|
||||
response="${RUNNER_TEMP}/nightly-candidate-dispatch-response-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.txt"
|
||||
status="$(curl -sS --retry 3 --connect-timeout 15 --max-time 60 \
|
||||
-o "${response}" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer ${DISPATCH_TOKEN}" \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary @"${body}" \
|
||||
https://api.github.com/repos/rustfs/auto-testing/dispatches)" || {
|
||||
echo "repository_dispatch request to rustfs/auto-testing failed before an HTTP status" >&2
|
||||
exit 1
|
||||
}
|
||||
if [[ "${status}" != "204" ]]; then
|
||||
echo "repository_dispatch to rustfs/auto-testing failed: HTTP ${status}" >&2
|
||||
if [[ -s "${response}" ]]; then
|
||||
cat "${response}" >&2
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
echo "dispatched nightly-candidate-ready to rustfs/auto-testing (deb_date=${DEB_DATE}, run=${GITHUB_RUN_ID}, attempt=${GITHUB_RUN_ATTEMPT})"
|
||||
# Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774).
|
||||
#
|
||||
# RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Validation Sweep
|
||||
|
||||
# Executes the migrated canonical validation cases (rustfs/auto-testing#113)
|
||||
# on the shared functional fleet. Everything with a lane executor is swept in
|
||||
# rotating nightly batches so the whole set is covered roughly weekly; a
|
||||
# single case or one module can be dispatched manually. That is the
|
||||
# implemented catalog (66 today) plus the eight planned cases whose dedicated
|
||||
# executors already exist and have live PASS evidence - fresh nightly
|
||||
# evidence feeds their acceptance review. heal.* cases stay on the
|
||||
# interactive validation flow: they require reviewed native/tool collectors
|
||||
# built by separate validation-tools runs before fleet execution.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# After the nightly functional chain settles (chain starts ~16:48 UTC and
|
||||
# runs for about two hours).
|
||||
- cron: "30 19 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: 'Single canonical validation case id (overrides module and rotation), e.g. pool.decommission'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
module:
|
||||
description: 'Restrict the sweep to one catalog module, e.g. bucket-replication'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
limit:
|
||||
description: 'Maximum cases per sweep (0 = run everything selected)'
|
||||
type: number
|
||||
required: false
|
||||
default: 12
|
||||
build_run_id:
|
||||
description: 'Successful nightly package run; empty resolves the latest'
|
||||
type: string
|
||||
required: false
|
||||
default: ''
|
||||
build_run_attempt:
|
||||
description: 'Attempt of that package run'
|
||||
type: string
|
||||
required: false
|
||||
default: '1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-validation-sweep
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES || 'vm000 vm001 vm002 vm003' }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER || 'azureuser' }}
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: Checkout chain tooling
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Resolve nightly candidate
|
||||
id: candidate
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
BUILD_RUN_ID: ${{ inputs.build_run_id }}
|
||||
BUILD_RUN_ATTEMPT: ${{ inputs.build_run_attempt }}
|
||||
CHAIN_OUTPUT: ${{ runner.temp }}/validation-sweep-candidate.json
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Scheduled runs carry no inputs and the resolver only derives a
|
||||
# build from a workflow_run payload (which a schedule cannot
|
||||
# provide), so resolve the newest successful nightly build
|
||||
# explicitly before invoking it.
|
||||
if [ -z "${BUILD_RUN_ID:-}" ]; then
|
||||
LATEST="$(gh api "repos/rustfs/rustfs/actions/workflows/nightly-gnu.yml/runs?branch=main&status=success&per_page=1" --jq '.workflow_runs[0]')"
|
||||
BUILD_RUN_ID="$(printf '%s' "${LATEST}" | jq -r '.id')"
|
||||
BUILD_RUN_ATTEMPT="$(printf '%s' "${LATEST}" | jq -r '.run_attempt')"
|
||||
export BUILD_RUN_ID BUILD_RUN_ATTEMPT
|
||||
echo "scheduled sweep pinned to nightly ${BUILD_RUN_ID} attempt ${BUILD_RUN_ATTEMPT}"
|
||||
fi
|
||||
python3 scripts/resolve_functional_candidate.py
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
root = Path(os.environ["RUNNER_TEMP"]) / "validation-sweep"
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
chain = json.loads((Path(os.environ["RUNNER_TEMP"]) / "validation-sweep-candidate.json").read_text())
|
||||
manifest = chain["candidate"]["manifest"]
|
||||
(root / "candidate.json").write_text(json.dumps({
|
||||
"package_url": manifest["package_url"],
|
||||
"package_sha256": manifest["package_sha256"],
|
||||
"source_sha": manifest["source_sha"],
|
||||
}, sort_keys=True) + "\n")
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write("testing_sha=" + chain["testing_sha"] + "\n")
|
||||
output.write("package_url=" + manifest["package_url"] + "\n")
|
||||
output.write("source_sha=" + manifest["source_sha"] + "\n")
|
||||
PY
|
||||
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: ${{ steps.candidate.outputs.testing_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
|
||||
- name: Prepare validation Python runtime
|
||||
# The pool / ilm / replication validation runners require a managed
|
||||
# Python, exactly like the chain's validation job provides.
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install validation Python requirements
|
||||
# Case scripts call python3 and import boto3 at startup. Install the
|
||||
# functional chain's client into a per-run env so that interpreter
|
||||
# sees it without changing the runner's shared Python.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
validation_python="${RUNNER_TEMP}/rustfs-validation-python-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
test ! -e "${validation_python}"
|
||||
python3 -m venv "${validation_python}"
|
||||
"${validation_python}/bin/python" -m pip install \
|
||||
-r auto-testing/scripts/validation_ilm/requirements.txt
|
||||
printf '%s\n' "${validation_python}/bin" >> "${GITHUB_PATH}"
|
||||
|
||||
- name: Select sweep cases
|
||||
id: select
|
||||
env:
|
||||
SELECTED_CASE: ${{ inputs.case }}
|
||||
SELECTED_MODULE: ${{ inputs.module }}
|
||||
CASE_LIMIT: ${{ inputs.limit }}
|
||||
SWEEP_EVENT: ${{ github.event_name }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
sweep_root = Path(os.environ["SWEEP_ROOT"])
|
||||
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
def sweepable(case):
|
||||
status = case.get("source_status")
|
||||
if status == "implemented":
|
||||
return True
|
||||
# The remaining planned non-heal cases route to the lane
|
||||
# executors and already have live PASS evidence; sweeping them
|
||||
# nightly produces fresh evidence for their acceptance review.
|
||||
return status == "planned" and not case["id"].startswith("heal.")
|
||||
|
||||
selected = [(case["id"], module["id"]) for module in catalog["modules"]
|
||||
for case in module["cases"]
|
||||
if sweepable(case)]
|
||||
if not selected:
|
||||
raise SystemExit("catalog has no sweepable cases")
|
||||
|
||||
one_case = os.environ.get("SELECTED_CASE", "").strip()
|
||||
module_filter = os.environ.get("SELECTED_MODULE", "").strip()
|
||||
known = {case_id for case_id, _ in selected}
|
||||
modules = {module_id for _, module_id in selected}
|
||||
if one_case:
|
||||
if one_case not in known:
|
||||
raise SystemExit(f"unknown case or no lane executor for it: {one_case}")
|
||||
selected = [(one_case, dict(selected)[one_case])]
|
||||
elif module_filter:
|
||||
if module_filter not in modules:
|
||||
raise SystemExit(f"unknown module or no implemented cases: {module_filter}")
|
||||
selected = [pair for pair in selected if pair[1] == module_filter]
|
||||
elif os.environ.get("SWEEP_EVENT") == "schedule":
|
||||
# Stateless rotation: every night take one window of the catalog
|
||||
# so a full sweep needs len(selected)/limit nights and no state
|
||||
# has to be carried between runs.
|
||||
limit = max(1, int(os.environ.get("CASE_LIMIT") or 12))
|
||||
day = datetime.datetime.now(datetime.timezone.utc).timetuple().tm_yday
|
||||
start = (day * limit) % len(selected)
|
||||
selected = (selected[start:] + selected[:start])[:limit]
|
||||
limit = int(os.environ.get("CASE_LIMIT") or 0)
|
||||
if one_case or module_filter or (not os.environ.get("SWEEP_EVENT") == "schedule"):
|
||||
if limit > 0:
|
||||
selected = selected[:limit]
|
||||
|
||||
(sweep_root / "sweep-cases.txt").write_text("".join(f"{case_id}\n" for case_id, _ in selected))
|
||||
with open(os.environ["GITHUB_OUTPUT"], "a") as output:
|
||||
output.write(f"count={len(selected)}\n")
|
||||
print(f"selected {len(selected)} case(s):", ", ".join(c for c, _ in selected))
|
||||
PY
|
||||
|
||||
- name: Admit clean shared environment
|
||||
id: environment
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
run: python3 scripts/functional_environment.py enter --suite heal
|
||||
|
||||
- name: Run validation cases
|
||||
id: run_cases
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p "${SWEEP_ROOT}/results"
|
||||
: > "${SWEEP_ROOT}/results.txt"
|
||||
while read -r CASE_ID; do
|
||||
[ -n "${CASE_ID}" ] || continue
|
||||
echo "::group::case ${CASE_ID}"
|
||||
rm -rf "${SWEEP_ROOT}/run"
|
||||
if ./auto-testing/scripts/run_validation_case.sh \
|
||||
"${CASE_ID}" "${SWEEP_ROOT}/candidate.json" \
|
||||
"${AUTO_TESTING_SHA}" "${SWEEP_ROOT}/run" \
|
||||
> "${SWEEP_ROOT}/results/${CASE_ID//\//_}.log" 2>&1; then
|
||||
printf '%s\tPASS\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: PASS"
|
||||
else
|
||||
printf '%s\tFAIL\n' "${CASE_ID}" >> "${SWEEP_ROOT}/results.txt"
|
||||
echo "case ${CASE_ID}: FAIL (log retained in the sweep artifact)"
|
||||
fi
|
||||
if [ -f "${SWEEP_ROOT}/run/report.md" ]; then
|
||||
cp "${SWEEP_ROOT}/run/report.md" "${SWEEP_ROOT}/results/${CASE_ID//\//_}.report.md" 2>/dev/null || true
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
# Fleet hygiene between cases: the case runners set up their own
|
||||
# topology, this purge guarantees the next case starts clean even
|
||||
# when the previous one died mid-run.
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
# -n keeps ssh from consuming the case list this loop reads from stdin
|
||||
ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || echo "inter-case cleanup failed on ${node} (continuing)"
|
||||
done
|
||||
done < "${SWEEP_ROOT}/sweep-cases.txt"
|
||||
echo "sweep finished: $(wc -l < "${SWEEP_ROOT}/results.txt") case(s) executed"
|
||||
|
||||
- name: Generate sweep report
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
PACKAGE_URL: ${{ steps.candidate.outputs.package_url }}
|
||||
SOURCE_SHA: ${{ steps.candidate.outputs.source_sha }}
|
||||
AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(os.environ["SWEEP_ROOT"])
|
||||
catalog = json.loads(Path("auto-testing/validation-cases.json").read_text())
|
||||
module_of = {case["id"]: module["id"] for module in catalog["modules"] for case in module["cases"]}
|
||||
rows = []
|
||||
for line in (root / "results.txt").read_text().splitlines():
|
||||
if not line:
|
||||
continue
|
||||
case_id, status = line.split("\t")
|
||||
rows.append((case_id, module_of.get(case_id, "?"), status))
|
||||
passed = sum(1 for _, _, s in rows if s == "PASS")
|
||||
failed = len(rows) - passed
|
||||
run_url = (f"{os.environ.get('GITHUB_SERVER_URL', 'https://github.com')}/"
|
||||
f"{os.environ.get('GITHUB_REPOSITORY', 'rustfs/rustfs')}/actions/runs/"
|
||||
f"{os.environ.get('GITHUB_RUN_ID', '')}")
|
||||
|
||||
report = [
|
||||
"# RustFS validation sweep report", "",
|
||||
f"- Run: {run_url}",
|
||||
f"- Attempt: {os.environ.get('GITHUB_RUN_ATTEMPT', '')}",
|
||||
f"- Trigger: {os.environ.get('GITHUB_EVENT_NAME', '')}",
|
||||
f"- Package: {os.environ.get('PACKAGE_URL', 'nightly (R2 latest)')}",
|
||||
f"- Source SHA: {os.environ.get('SOURCE_SHA', '')}",
|
||||
f"- auto-testing: {os.environ.get('AUTO_TESTING_SHA', '')}", "",
|
||||
"Canonical validation cases (rustfs/auto-testing#113) executed by the sweep, grouped by catalog module.", "",
|
||||
]
|
||||
# Group the executed cases by catalog module (the major category:
|
||||
# pool lifecycle, heal, ILM expiration, bucket/site replication, kms,
|
||||
# data services, identity, s3 compatibility). Per-module subtotals
|
||||
# deliberately use "- Subtotal:" so the dashboard overview regexes
|
||||
# (which anchor on "- Total:") keep reading the grand totals below.
|
||||
by_module = {}
|
||||
module_order = []
|
||||
for case_id, module, status in rows:
|
||||
if module not in by_module:
|
||||
by_module[module] = []
|
||||
module_order.append(module)
|
||||
by_module[module].append((case_id, status))
|
||||
report += ["## Case Summary", ""]
|
||||
for module in module_order:
|
||||
cases = by_module[module]
|
||||
mod_pass = sum(1 for _, st in cases if st == "PASS")
|
||||
mod_fail = len(cases) - mod_pass
|
||||
report += [f"### {module} ({len(cases)})", "",
|
||||
"| Case | Status |", "| --- | --- |"]
|
||||
report += [f"| {case_id} | {status} |" for case_id, status in cases]
|
||||
report += ["", f"- Subtotal: {mod_pass} passed, {mod_fail} failed", ""]
|
||||
report += [
|
||||
"## Totals", "",
|
||||
f"- Total: {len(rows)}",
|
||||
f"- PASS: {passed}",
|
||||
f"- FAIL: {failed}",
|
||||
"- UNSUPPORTED: 0", "",
|
||||
"Failing cases keep this workflow red only as a report; the sweep artifact",
|
||||
"retains every case log and per-case report for diagnosis.", "",
|
||||
]
|
||||
(root / "report.md").write_text("\n".join(report))
|
||||
(root / "cases.md").write_text("\n".join(f"| {c} | {m} | {s} |" for c, m, s in rows) + "\n")
|
||||
PY
|
||||
cat "${SWEEP_ROOT}/report.md" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 2
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: ${{ runner.temp }}/validation-sweep/report.md
|
||||
SUITE: validation
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
B64_FILE="$(mktemp)"
|
||||
python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \
|
||||
'{message:$msg, content:($content|rtrimstr("\n"))}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
echo "dashboard updated: ${REPORT_PATH}"
|
||||
|
||||
- name: Retain sweep evidence
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: rustfs-validation-sweep-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ runner.temp }}/validation-sweep/report.md
|
||||
${{ runner.temp }}/validation-sweep/results.txt
|
||||
${{ runner.temp }}/validation-sweep/sweep-cases.txt
|
||||
${{ runner.temp }}/validation-sweep/candidate.json
|
||||
${{ runner.temp }}/validation-sweep/results/
|
||||
${{ runner.temp }}/validation-sweep/run/
|
||||
# warn (not error): an early infra failure legitimately leaves no
|
||||
# per-case evidence, and the failing step is already red.
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
id: environment_cleanup
|
||||
timeout-minutes: 5
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
cleanup_status=0
|
||||
for node in ${RUSTFS_NODES}; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -f /etc/default/rustfs
|
||||
' || cleanup_status=$?
|
||||
done
|
||||
exit "${cleanup_status}"
|
||||
|
||||
- name: Release environment only after successful cleanup
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-sweep.json
|
||||
CLEANUP_OUTCOME: ${{ steps.environment_cleanup.outcome }}
|
||||
run: python3 scripts/functional_environment.py finish --suite heal --cleanup-outcome "$CLEANUP_OUTCOME"
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: echo "RustFS validation sweep failed (harness/environment breakdown); see the sweep artifact for case logs."
|
||||
|
||||
- name: Enforce sweep result
|
||||
if: ${{ always() && steps.environment.outcome == 'success' }}
|
||||
env:
|
||||
SWEEP_ROOT: ${{ runner.temp }}/validation-sweep
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if grep -q $'\tFAIL$' "${SWEEP_ROOT}/results.txt"; then
|
||||
echo "validation sweep has failing cases:"
|
||||
grep $'\tFAIL$' "${SWEEP_ROOT}/results.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
alert-on-failure:
|
||||
name: Alert on scheduled failure
|
||||
# A scheduled sweep failure usually means a canonical validation case
|
||||
# regressed or the fleet environment broke; surface it as a tracking
|
||||
# issue instead of leaving it in the Actions tab.
|
||||
needs: [sweep]
|
||||
if: github.repository == 'rustfs/rustfs' && (always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v5.0.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Open or update failure-tracking issue
|
||||
uses: ./.github/actions/schedule-failure-issue
|
||||
with:
|
||||
github-token: ${{ secrets.BACKLOG_ISSUE_TOKEN }}
|
||||
@@ -107,6 +107,32 @@ jobs:
|
||||
shell: pwsh
|
||||
run: cargo test -p rustfs-ecstore --lib multipart_owned_record_parent_barriers_ -- --nocapture
|
||||
|
||||
# Native multipart completion publishes its payload from a different
|
||||
# parent than its metadata, and abort renames under a precondition; both
|
||||
# need their own Windows handle discipline.
|
||||
- name: Test native multipart complete and abort publication
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_abort_missing_state_with_actual_owned_records_never_becomes_legacy_or_missing
|
||||
native_abort_pending_intent_is_not_a_delete_certificate_and_raw_cleanup_preserves_exact_evidence
|
||||
--nocapture
|
||||
|
||||
# Owned-undo reclaim removes a directory it pinned, copy heal persists
|
||||
# fault bytes through a handle, and metadata reads meet a directory.
|
||||
- name: Test native multipart reclaim, copy heal, and directory reads
|
||||
shell: pwsh
|
||||
run: >-
|
||||
cargo test -p rustfs-ecstore --lib --
|
||||
native_multipart_owned_undo
|
||||
native_multipart_copy_heal
|
||||
orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk
|
||||
--nocapture
|
||||
|
||||
- name: Test directory removal fallback
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-ecstore --lib disk::fs::tests::test_remove -- --nocapture"
|
||||
|
||||
- name: Test Windows path conversion
|
||||
shell: pwsh
|
||||
run: "cargo test -p rustfs-utils --no-default-features --features path --lib dunce:: -- --nocapture"
|
||||
|
||||
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- IAM snapshot/deletion replay, target-assigned delete-marker purges, timestamp ordering, and best-effort peer broadcast now close the control-plane gaps found by the R6 review ([#7195](https://github.com/rustfs/rustfs/pull/7195)).
|
||||
- A `set-remote-target?update=true&creds=true` request on a site-replication peer's bucket target is now rejected with `400 InvalidRequest`, naming the target and why it counts as peer-owned. It used to return success while keeping the stored endpoint, credentials, and TLS settings, so an endpoint edit from the Console's bucket replication form appeared to apply but did not. Change a peer's address with a site-replication edit instead; see "Changing a site's address" in `docs/operations/site-replication-operations.md`. The `sync`, `bandwidth`, `proxy`, and `path` update groups are unchanged.
|
||||
- Site-replication edits that omit the site name, such as `mc admin replicate update --deployment-id <ID> --endpoint <URL>`, now keep the stored name on the editing site and on every receiver instead of renaming the site after the new endpoint host. No persisted format changes.
|
||||
- The persisted replication MRF journal is now replayed every 5 minutes while the server runs, not only at startup. Delete-marker purges whose targets were unreachable, and replications that overflowed the worker queues, now converge within one replay interval after the target returns instead of waiting for a restart. A target that stays down is re-attempted once per retained entry per interval across the cluster: the start time of the last pass is shared through a small `config/replication/mrf.bin.replay` object, so nodes do not each replay once per interval. Older versions ignore that object; existing persisted formats are unchanged.
|
||||
- Upgrade and rollback: upgrade every node in one site consecutively and verify reconciliation before moving to the next site; do not intentionally run a site mixed-version. Target-version ledger keys are harmless on rollback, although old code cannot use their routing. Before rolling back past [#7307](https://github.com/rustfs/rustfs/pull/7307), drain or repair every pending version purge: older code can free a retained version's data directory before its remote purge is acknowledged. See `docs/operations/site-replication-operations.md`.
|
||||
|
||||
### Security
|
||||
|
||||
Generated
+399
-195
File diff suppressed because it is too large
Load Diff
+15
-17
@@ -147,7 +147,7 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
async-channel = "2.5.0"
|
||||
async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.51" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
@@ -168,7 +168,7 @@ http-body-util = "0.1.5"
|
||||
memchr = "2.8.3"
|
||||
minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
rustfs-kafka-async = { version = "1.4.1" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.54.1" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
@@ -191,6 +191,7 @@ quinn = { version = "0.11.12", default-features = false, features = [
|
||||
# Serialization and Data Formats
|
||||
apache-avro = { version = "0.22.0", features = ["snappy", "zstandard"] }
|
||||
bytes = { version = "1.12.1" }
|
||||
csv-core = "0.1.13"
|
||||
bytesize = "2.7.0"
|
||||
byteorder = "1.5.0"
|
||||
flatbuffers = "25.12.19"
|
||||
@@ -212,12 +213,13 @@ serde_urlencoded = "0.7.1"
|
||||
# releases.
|
||||
aes-gcm = { version = "0.11.1" }
|
||||
argon2 = { version = "0.6.0" }
|
||||
aws-lc-rs = { version = "1.19.0", default-features = false }
|
||||
blake2 = "0.11.0"
|
||||
chacha20poly1305 = { version = "0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
hmac = { version = "0.13.0" }
|
||||
jsonwebtoken = { version = "11.1.0" }
|
||||
openidconnect = { default-features = false, version = "4.0" }
|
||||
openidconnect = { default-features = false, version = "4.1.0" }
|
||||
pbkdf2 = "0.13.0"
|
||||
p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
|
||||
rsa = { version = "=0.10.0-rc.18" }
|
||||
@@ -234,7 +236,7 @@ proptest = "1"
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.38" }
|
||||
jiff = { version = "0.2.39" }
|
||||
time = { version = "0.3.55" }
|
||||
|
||||
# Database
|
||||
@@ -253,9 +255,9 @@ atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
aws-credential-types = { version = "1.3.1" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.128.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.156.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.123.0" }
|
||||
aws-sdk-kms = { default-features = false, version = "1.129.0" }
|
||||
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
|
||||
aws-sdk-sts = { default-features = false, version = "1.124.0" }
|
||||
aws-smithy-async = { version = "1.3.1" }
|
||||
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
|
||||
aws-smithy-runtime-api = { version = "1.19.2" }
|
||||
@@ -337,7 +339,7 @@ strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.0"
|
||||
test-case = "3.4.1"
|
||||
thiserror = "2.0.21"
|
||||
tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
@@ -356,13 +358,13 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "9.1.3"
|
||||
zip = "9.3.0"
|
||||
zstd = "0.14.3"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.6", default-features = false }
|
||||
dial9 = { version = "0.5.7", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
@@ -377,16 +379,16 @@ libunftp = { version = "0.24.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.2.1" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.65.0" }
|
||||
russh = { version = "0.65.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.2", default-features = false }
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
@@ -427,7 +429,3 @@ strip = "none"
|
||||
# Pin upstream fixes until compatible crates.io releases are available.
|
||||
[patch.crates-io]
|
||||
zstd = { path = "crates/zstd-compat" }
|
||||
# Preserve RustFS OIDC flows while migrating the old elliptic-curve dependency chain.
|
||||
# Upstream: openidconnect-rs#244 and oauth2-rs#352.
|
||||
openidconnect = { git = "https://github.com/ramosbugs/openidconnect-rs", rev = "2f77ee73bc4554e1cb6279adb386005429fb4e41" }
|
||||
oauth2 = { git = "https://github.com/ramosbugs/oauth2-rs", rev = "89585f4a280afd7b285eaf1c9be0068874e03d22" }
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM alpine:3.24.2 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl jq unzip
|
||||
WORKDIR /build
|
||||
@@ -69,6 +70,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
|
||||
|
||||
|
||||
FROM alpine:3.24.2
|
||||
|
||||
@@ -95,14 +99,18 @@ RUN apk upgrade --no-cache && \
|
||||
ca-certificates \
|
||||
coreutils \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN addgroup -g 10001 -S rustfs && \
|
||||
adduser -u 10001 -G rustfs -S rustfs -D && \
|
||||
@@ -124,6 +132,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM rust:1.97.1-trixie
|
||||
FROM rust:1.99.1-trixie
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN set -eux; \
|
||||
export DEBIAN_FRONTEND=noninteractive; \
|
||||
@@ -21,6 +24,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -36,6 +40,7 @@ COPY . .
|
||||
RUN ./scripts/static.sh
|
||||
RUN cargo run --bin gproto
|
||||
RUN cargo build --release --locked --bin rustfs
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
RUN set -eux; \
|
||||
groupadd -g 10001 rustfs; \
|
||||
@@ -50,8 +55,12 @@ RUN set -eux; \
|
||||
chown -R rustfs:rustfs /data /logs /app; \
|
||||
chmod 0750 /data /logs
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/src/rustfs/target/release/rustfs /usr/bin/rustfs && \
|
||||
install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
RUST_LOG="warn" \
|
||||
@@ -63,5 +72,5 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
|
||||
EXPOSE 9000 9001
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
+11
-3
@@ -16,6 +16,7 @@ FROM ubuntu:26.04 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
@@ -76,6 +77,9 @@ RUN set -eux; \
|
||||
chmod +x /build/rustfs; \
|
||||
rm -rf rustfs.zip /build/.tmp || true
|
||||
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
|
||||
|
||||
FROM ubuntu:26.04
|
||||
|
||||
ARG RELEASE=latest
|
||||
@@ -99,14 +103,18 @@ RUN apt-get update && apt-get upgrade -y \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530" \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
RUN groupadd -g 10001 rustfs && \
|
||||
useradd -u 10001 -g rustfs -m -s /sbin/nologin rustfs && \
|
||||
@@ -131,6 +139,6 @@ VOLUME ["/data"]
|
||||
|
||||
USER rustfs
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
|
||||
CMD ["rustfs"]
|
||||
|
||||
+19
-7
@@ -28,17 +28,19 @@ ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES=""
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
# -----------------------------
|
||||
# Build stage
|
||||
# -----------------------------
|
||||
FROM rust:1.97.1-trixie AS builder
|
||||
FROM rust:1.99.1-trixie AS builder
|
||||
|
||||
# Re-declare args after FROM
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES
|
||||
ARG RC_RELEASE
|
||||
|
||||
# Debug: print platforms
|
||||
RUN echo "Build info -> BUILDPLATFORM=${BUILDPLATFORM}, TARGETPLATFORM=${TARGETPLATFORM}"
|
||||
@@ -52,6 +54,7 @@ RUN set -eux; \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
git \
|
||||
pkg-config \
|
||||
libssl-dev \
|
||||
@@ -166,6 +169,8 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
# -----------------------------
|
||||
# Development stage (keeps toolchain)
|
||||
# -----------------------------
|
||||
@@ -199,10 +204,13 @@ ENV RUSTFS_VOLUMES="/data" \
|
||||
# We rely on cargo run to build and run
|
||||
EXPOSE 9000 9001
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN install -m 0755 /usr/local/bin/rc /usr/bin/rc && \
|
||||
chmod +x /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["cargo", "run", "--bin", "rustfs", "--"]
|
||||
|
||||
# -----------------------------
|
||||
@@ -227,6 +235,7 @@ RUN set -eux; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
jq \
|
||||
tzdata \
|
||||
coreutils; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
@@ -248,8 +257,11 @@ RUN set -eux; \
|
||||
|
||||
# Copy the freshly built binary and the entrypoint
|
||||
COPY --from=builder /usr/local/bin/rustfs /usr/bin/rustfs
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /usr/bin/rustfs /entrypoint.sh
|
||||
COPY --from=builder /usr/local/bin/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
COPY bootstrap/ /etc/rustfs/bootstrap/
|
||||
RUN chmod +x /usr/bin/rustfs /usr/bin/rc /etc/rustfs/entrypoint.sh && \
|
||||
ln -s /etc/rustfs/entrypoint.sh /entrypoint.sh
|
||||
|
||||
# Default environment (override in docker run/compose as needed)
|
||||
ENV RUSTFS_VOLUMES="/data" \
|
||||
@@ -263,5 +275,5 @@ EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
# Keep root here; entrypoint will drop privileges using chroot --userspec
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
ENTRYPOINT ["/etc/rustfs/entrypoint.sh"]
|
||||
CMD ["/usr/bin/rustfs"]
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# RustFS image bootstrap
|
||||
|
||||
The official image bundles this directory at `/etc/rustfs/bootstrap`. The default `buckets.json` is an empty JSON array, so the image starts RustFS directly and does not require a bootstrap endpoint or bootstrap credentials.
|
||||
|
||||
To create buckets, replace `buckets.json` in a derived image with a JSON array of bucket names. The entrypoint creates each bucket if it does not exist and applies `public-access-block.json`, whose default blocks all four public access paths. Edit that file in the derived image only when public access is an explicit requirement.
|
||||
|
||||
Optional per-bucket files use the bucket name as the filename without an extension:
|
||||
|
||||
| Directory | File format | Applied operation |
|
||||
| --- | --- | --- |
|
||||
| `policies/` | S3 bucket policy JSON | `rc bucket anonymous set-json` |
|
||||
| `versioning/` | JSON boolean (`true` or `false`) | Enable or suspend bucket versioning |
|
||||
| `lifecycle/` | `rc bucket lifecycle rule import` JSON | Replace lifecycle rules |
|
||||
| `cors/` | JSON or XML accepted by `rc bucket cors set` | Replace CORS rules |
|
||||
| `notifications/` | JSON array of `{ "arn": "...", "events": ["..."] }` | Add or replace event rules by ARN (prefix/suffix filters are not currently accepted) |
|
||||
|
||||
Every per-bucket file must correspond to a bucket listed in `buckets.json`. Applying these files is repeatable: bucket creation ignores existing buckets, while each configuration operation replaces or reconciles that bucket's requested state.
|
||||
|
||||
When `buckets.json` is non-empty, set `RUSTFS_BOOTSTRAP_ENDPOINT` to the RustFS S3 endpoint reachable from the container, for example `http://rustfs:9000` for a Compose service or the internal service DNS name in Kubernetes. The entrypoint waits on that endpoint's `/health/ready`; it never assumes `localhost`. `RUSTFS_BOOTSTRAP_TIMEOUT` controls the readiness wait in seconds (default `120`, maximum `3600`). `RUSTFS_BOOTSTRAP_DIR` can point to a different in-image configuration directory; it defaults to `/etc/rustfs/bootstrap`. Set `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY`, or their `_FILE` variants, for bootstrap authentication. The endpoint and credentials are read at runtime and are not baked into the image.
|
||||
|
||||
RustFS built-in IAM policies remain unchanged; the default bootstrap installs no custom bucket policies, lifecycle rules, versioning, CORS rules, or notification rules. The bundled RustFS CLI is resolved from a published release and its release checksum is verified. Set the Docker build argument `RC_RELEASE` to a version to pin it. Build the RustFS image after a CLI release containing the `bucket public-access-block` commands is available. For Kubernetes, the configured service must route traffic to the starting pod before its readiness probe succeeds (for example, with `publishNotReadyAddresses: true`); otherwise the service endpoint cannot reach the pod while the entrypoint waits for readiness.
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"block_public_acls": true,
|
||||
"ignore_public_acls": true,
|
||||
"block_public_policy": true,
|
||||
"restrict_public_buckets": true
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -30,10 +30,10 @@ pub const ENV_TRUSTED_PROXY_VALIDATION_MODE: &str = "RUSTFS_TRUSTED_PROXY_VALIDA
|
||||
/// Default validation mode is "hop_by_hop".
|
||||
pub const DEFAULT_TRUSTED_PROXY_VALIDATION_MODE: &str = "hop_by_hop";
|
||||
|
||||
/// Environment variable to enable RFC 7239 "Forwarded" header support.
|
||||
/// Select RFC 7239 "Forwarded" instead of "X-Forwarded-For" in legacy mode.
|
||||
pub const ENV_TRUSTED_PROXY_ENABLE_RFC7239: &str = "RUSTFS_TRUSTED_PROXY_ENABLE_RFC7239";
|
||||
/// RFC 7239 support is enabled by default.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = true;
|
||||
/// Legacy mode uses X-Forwarded-For unless RFC 7239 is explicitly selected.
|
||||
pub const DEFAULT_TRUSTED_PROXY_ENABLE_RFC7239: bool = false;
|
||||
|
||||
/// Environment variable for the maximum allowed proxy hops.
|
||||
pub const ENV_TRUSTED_PROXY_MAX_HOPS: &str = "RUSTFS_TRUSTED_PROXY_MAX_HOPS";
|
||||
|
||||
@@ -146,3 +146,9 @@ pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
||||
|
||||
/// Queue-store compression is enabled by default to reduce disk footprint.
|
||||
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
|
||||
|
||||
/// Maximum on-disk bytes in each target queue, including failed entries and pending writes.
|
||||
pub const ENV_TARGET_STORE_MAX_BYTES: &str = "RUSTFS_TARGET_STORE_MAX_BYTES";
|
||||
|
||||
/// Queue writes fail at one GiB unless the operator sets another positive byte budget.
|
||||
pub const DEFAULT_TARGET_STORE_MAX_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
@@ -175,12 +175,17 @@ pub const ENV_HTTP_SOCKET_RECV_BUFFER_BYTES: &str = "RUSTFS_HTTP_SOCKET_RECV_BUF
|
||||
pub const DEFAULT_HTTP_SOCKET_RECV_BUFFER_BYTES: usize = 0;
|
||||
|
||||
/// Environment variable for the S3 request-body inter-chunk read timeout
|
||||
/// (seconds). Default: 300. Set to 0 to disable.
|
||||
/// (seconds). Default: 300. Set to 0 to disable the inactivity timeout.
|
||||
///
|
||||
/// This bounds how long a `PutObject`/upload may wait for the *next* body chunk
|
||||
/// while more bytes are still expected. It resets on every chunk, so it does not
|
||||
/// penalize slow-but-progressing uploads — it only fires when a peer sends a
|
||||
/// partial body and then goes silent *without* closing the connection (no EOF).
|
||||
/// Independently of this knob, `PutObject` and `UploadPart` require at least
|
||||
/// 64 KiB of body progress per 60 seconds of server wait (the minimum-progress
|
||||
/// floor, logged as `upload_body_progress_stalled`); it bounds trickle uploads
|
||||
/// that would otherwise pin upload admission indefinitely, and it stays in
|
||||
/// force when this timeout is set to 0.
|
||||
/// A reverse proxy or CDN that forwards a truncated body this way would
|
||||
/// otherwise make RustFS wait forever for bytes that never arrive; with this
|
||||
/// timeout the stalled read is aborted and logged with the received/expected
|
||||
|
||||
@@ -26,7 +26,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **protocols** | [`src/protocols/`](src/protocols) | FTPS, WebDAV, SFTP compliance. Fixed ports, own guide: [`src/protocols/README.md`](src/protocols/README.md) |
|
||||
| **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) |
|
||||
| **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test`, `tier_stats_cluster_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Storage-sensitive PR and nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
|
||||
@@ -194,9 +194,9 @@ the same profile for membership and execution with one nightly worker.
|
||||
| `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) |
|
||||
| ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) |
|
||||
| KMS suite | `e2e-full` job, merge queue + main | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** |
|
||||
| Direct and mixed-version rolling upgrades from pinned previous release | `e2e-upgrade.yml`, release tags + weekly schedule + manual dispatch | **Active** |
|
||||
| Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (storage-sensitive PR / nightly / dispatch) |
|
||||
| Distributed 4-node 4-disk (`e2e-distributed` profile) | `.github/workflows/e2e-distributed.yml` | **Active** (nightly / dispatch) |
|
||||
| Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) |
|
||||
| Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) |
|
||||
| Replication (slow + multi-node) | `e2e-repl-nightly` profile, consolidated nightly workflow | **Active** (backlog#1147 repl-1) |
|
||||
|
||||
@@ -502,10 +502,163 @@ mod tests {
|
||||
assert!(response.contains(if expected == reqwest::StatusCode::FORBIDDEN {
|
||||
"<Code>AccessDenied</Code>"
|
||||
} else {
|
||||
"<Code>BucketAlreadyExists</Code>"
|
||||
"<Code>BucketAlreadyOwnedByYou</Code>"
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A deployment owns a single bucket namespace, so an IAM user that recreates
|
||||
/// an existing bucket is the owner, not another account. In the default region
|
||||
/// every such request returns the legacy idempotent `200 OK` (AWS CreateBucket
|
||||
/// semantics), including buckets created by root and by another IAM user.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_in_default_region_is_idempotent_for_iam_user()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "us-east-1"]).await?;
|
||||
let root = env.create_s3_client();
|
||||
|
||||
admin_create_user(&env, "duplicate-iam", "duplicate-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-iam",
|
||||
"duplicate-iam-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
));
|
||||
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root creates the first bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-root-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("an IAM user recreating a root-owned bucket in us-east-1 gets 200 OK");
|
||||
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the IAM user creates its own bucket");
|
||||
iam.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("the same IAM user recreating its bucket gets 200 OK");
|
||||
root.create_bucket()
|
||||
.bucket("duplicate-iam-owned")
|
||||
.send()
|
||||
.await
|
||||
.expect("root recreating an IAM-created bucket gets 200 OK");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Outside the default region the same ownership answer becomes
|
||||
/// `409 BucketAlreadyOwnedByYou` for every authorized requester. Requests that
|
||||
/// lack `s3:CreateBucket` still fail authorization before the duplicate check.
|
||||
#[tokio::test]
|
||||
async fn test_create_bucket_duplicate_outside_default_region_reports_owned_by_you() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec!["--region", "eu-west-1"]).await?;
|
||||
let region = "eu-west-1";
|
||||
let root = Client::from_conf(
|
||||
env.create_s3_client()
|
||||
.config()
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-iam", "duplicate-eu-iam-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readwrite",
|
||||
"duplicate-eu-iam",
|
||||
)
|
||||
.await?;
|
||||
let iam = Client::from_conf(
|
||||
build_test_s3_config(&env.url, "duplicate-eu-iam", "duplicate-eu-iam-secret", None, "duplicate-create")
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
admin_create_user(&env, "duplicate-eu-readonly", "duplicate-eu-readonly-secret").await?;
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"readonly",
|
||||
"duplicate-eu-readonly",
|
||||
)
|
||||
.await?;
|
||||
let readonly = Client::from_conf(
|
||||
build_test_s3_config(
|
||||
&env.url,
|
||||
"duplicate-eu-readonly",
|
||||
"duplicate-eu-readonly-secret",
|
||||
None,
|
||||
"duplicate-create",
|
||||
)
|
||||
.to_builder()
|
||||
.region(Region::new(region))
|
||||
.build(),
|
||||
);
|
||||
|
||||
for (client, bucket) in [(&root, "duplicate-eu-root"), (&iam, "duplicate-eu-iam")] {
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{bucket} must be created: {error:?}"));
|
||||
// Both the creator and the other authenticated identity own the namespace.
|
||||
for creator in [&root, &iam] {
|
||||
let error = creator
|
||||
.create_bucket()
|
||||
.bucket(bucket)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a duplicate create outside us-east-1 must report ownership");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("BucketAlreadyOwnedByYou"),
|
||||
"{bucket}"
|
||||
);
|
||||
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(409), "{bucket}");
|
||||
}
|
||||
}
|
||||
|
||||
let denied = readonly
|
||||
.create_bucket()
|
||||
.bucket("duplicate-eu-root")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a requester without s3:CreateBucket must be denied before the duplicate check");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,11 +12,249 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, assert_object_bytes, payload_for, put_object, unique_bucket};
|
||||
use super::harness::{DistCluster, DistLayout, TestResult, payload_for, s3_probe_client, sha256_hex, unique_bucket};
|
||||
use crate::common::init_logging;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::{ProvideErrorMetadata, SdkError},
|
||||
primitives::ByteStream,
|
||||
};
|
||||
use std::any::TypeId;
|
||||
use std::collections::BTreeSet;
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Barrier;
|
||||
use tokio::time::Instant;
|
||||
|
||||
const LOCK_ADMISSION_BUDGET: Duration = Duration::from_secs(45);
|
||||
|
||||
async fn after_lock_contention<T, E, F, Fut>(operation: &str, mut call: F) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
E: Debug + ProvideErrorMetadata + 'static,
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<T, SdkError<E>>>,
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut attempts = 0;
|
||||
// The workload uses unique unversioned keys. Repeat only explicitly
|
||||
// advertised lock backpressure; every step still needs its real result.
|
||||
let outcome = tokio::time::timeout(LOCK_ADMISSION_BUDGET, async {
|
||||
loop {
|
||||
attempts += 1;
|
||||
let result = call().await;
|
||||
let retry = result.as_ref().err().is_some_and(|error| {
|
||||
let Some(response) = error.raw_response() else {
|
||||
return false;
|
||||
};
|
||||
let Some(service_error) = error.as_service_error() else {
|
||||
return false;
|
||||
};
|
||||
// HEAD has no XML error body, so its SDK service error can
|
||||
// lack a code even when the server supplies retry headers.
|
||||
let namespace_code = service_error.code() == Some("ServiceUnavailable")
|
||||
|| (service_error.code().is_none()
|
||||
&& TypeId::of::<E>() == TypeId::of::<aws_sdk_s3::operation::head_object::HeadObjectError>());
|
||||
response.status().as_u16() == 503
|
||||
&& namespace_code
|
||||
&& response.headers().get("x-rustfs-error-cause") == Some("namespace_lock_timeout")
|
||||
&& response.headers().get("retry-after") == Some("1")
|
||||
});
|
||||
if !retry {
|
||||
return result;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
if attempts > 1 || !matches!(&outcome, Ok(Ok(_))) {
|
||||
eprintln!(
|
||||
"mixed workload request: operation={operation}, attempts={attempts}, elapsed={:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
}
|
||||
outcome.map_err(|_| {
|
||||
format!("{operation} exceeded its {LOCK_ADMISSION_BUDGET:?} request budget after {attempts} attempts").into()
|
||||
})
|
||||
}
|
||||
|
||||
async fn assert_workload_bytes(client: &Client, bucket: &str, key: &str, expected: &[u8]) -> TestResult {
|
||||
let output = after_lock_contention(&format!("GET {key}"), || client.get_object().bucket(bucket).key(key).send()).await??;
|
||||
let actual = output.body.collect().await?.into_bytes();
|
||||
if actual.as_ref() != expected {
|
||||
return Err(format!(
|
||||
"object {key} bytes mismatch: expected {} bytes sha256={} got {} bytes sha256={}",
|
||||
expected.len(),
|
||||
sha256_hex(expected),
|
||||
actual.len(),
|
||||
sha256_hex(&actual)
|
||||
)
|
||||
.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aws_smithy_http_client::test_util::infallible_client_fn;
|
||||
use aws_smithy_types::body::SdkBody;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
type Reply = (u16, &'static str, Option<&'static str>, Option<&'static str>, &'static str);
|
||||
|
||||
fn client(respond: impl Fn(usize) -> Reply + Send + Sync + 'static) -> (Client, Arc<AtomicUsize>) {
|
||||
let requests = Arc::new(AtomicUsize::new(0));
|
||||
let seen = Arc::clone(&requests);
|
||||
let transport = infallible_client_fn(move |_| {
|
||||
let (status, code, cause, retry_after, body) = respond(seen.fetch_add(1, Ordering::SeqCst));
|
||||
let body = if code.is_empty() {
|
||||
body.to_string()
|
||||
} else {
|
||||
format!("<Error><Code>{code}</Code></Error>")
|
||||
};
|
||||
let mut response = http::Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/xml")
|
||||
.header("content-length", body.len());
|
||||
if let Some(cause) = cause {
|
||||
response = response.header("x-rustfs-error-cause", cause);
|
||||
}
|
||||
if let Some(retry_after) = retry_after {
|
||||
response = response.header("retry-after", retry_after);
|
||||
}
|
||||
response.body(SdkBody::from(body)).expect("scripted S3 response")
|
||||
});
|
||||
let config = crate::common::build_test_s3_config(
|
||||
"http://contention.example.invalid",
|
||||
"synthetic-access",
|
||||
"synthetic-secret",
|
||||
None,
|
||||
"contention-test",
|
||||
);
|
||||
let client = Client::from_conf(config.to_builder().http_client(transport).build());
|
||||
(s3_probe_client(&client), requests)
|
||||
}
|
||||
|
||||
fn contention() -> Reply {
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn advertised_contention_recovers_after_standard_sdk_attempts_are_exhausted() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt < 3 {
|
||||
contention()
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
let standard = Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(3))
|
||||
.build(),
|
||||
);
|
||||
standard
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("three advertised lock responses exhaust the standard SDK policy");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 3);
|
||||
requests.store(0, Ordering::SeqCst);
|
||||
let started = Instant::now();
|
||||
after_lock_contention("PUT recovery", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 4);
|
||||
assert!(started.elapsed() >= Duration::from_secs(3), "honor each Retry-After response");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn bodyless_head_contention_uses_explicit_retry_headers() -> TestResult {
|
||||
let (client, requests) = client(|attempt| {
|
||||
if attempt == 0 {
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), "")
|
||||
} else {
|
||||
(200, "", None, None, "")
|
||||
}
|
||||
});
|
||||
after_lock_contention("HEAD recovery", || client.head_object().bucket("bucket").key("key").send()).await??;
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn unrelated_or_malformed_responses_are_not_retried() -> TestResult {
|
||||
for reply in [
|
||||
(500, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(403, "AccessDenied", None, None, ""),
|
||||
(404, "NoSuchKey", None, None, ""),
|
||||
(503, "ServiceUnavailable", Some("storage_write_quorum"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_unavailable"), Some("1"), ""),
|
||||
(503, "InternalError", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "", Some("namespace_lock_timeout"), Some("1"), ""),
|
||||
(503, "ServiceUnavailable", None, Some("1"), ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), None, ""),
|
||||
(503, "ServiceUnavailable", Some("namespace_lock_timeout"), Some("2"), ""),
|
||||
] {
|
||||
let (client, requests) = client(move |_| reply);
|
||||
let result = after_lock_contention("PUT fatal response", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await?;
|
||||
assert!(result.is_err(), "fatal response cannot become success: {reply:?}");
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1, "fatal response must remain visible: {reply:?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn persistent_lock_contention_exhausts_the_request_budget() {
|
||||
let (client, requests) = client(|_| contention());
|
||||
let started = Instant::now();
|
||||
let result = after_lock_contention("PUT unavailable", || {
|
||||
client
|
||||
.put_object()
|
||||
.bucket("bucket")
|
||||
.key("key")
|
||||
.body(ByteStream::from_static(b"body"))
|
||||
.send()
|
||||
})
|
||||
.await;
|
||||
assert!(result.is_err(), "persistent contention cannot acknowledge an operation");
|
||||
assert!(started.elapsed() >= LOCK_ADMISSION_BUDGET);
|
||||
assert!(requests.load(Ordering::SeqCst) > 1);
|
||||
assert!(requests.load(Ordering::SeqCst) <= LOCK_ADMISSION_BUDGET.as_secs() as usize + 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_wrong_bytes_fail_without_retry() {
|
||||
let (client, requests) = client(|_| (200, "", None, None, "wrong bytes"));
|
||||
let error = assert_workload_bytes(&client, "bucket", "key", b"expected bytes")
|
||||
.await
|
||||
.expect_err("a successful but incorrect read is a consistency failure");
|
||||
assert!(error.to_string().contains("bytes mismatch"));
|
||||
assert_eq!(requests.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node() -> TestResult {
|
||||
@@ -24,7 +262,9 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let dist = DistCluster::start(DistLayout::FourByFour).await?;
|
||||
let bucket = unique_bucket("conc");
|
||||
dist.create_bucket(&bucket).await?;
|
||||
let clients = Arc::new(dist.clients()?);
|
||||
// This consistency probe owns its admission policy; unrelated responses
|
||||
// must not be hidden by the SDK's general-purpose retries.
|
||||
let clients = Arc::new(dist.clients()?.iter().map(s3_probe_client).collect::<Vec<_>>());
|
||||
let worker_count = 24;
|
||||
let rounds = 4;
|
||||
let barrier = Arc::new(Barrier::new(worker_count));
|
||||
@@ -44,31 +284,46 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
let key = format!("source/worker-{idx:02}-round-{round}.bin");
|
||||
let copy_key = format!("retained/worker-{idx:02}-round-{round}.bin");
|
||||
let body = payload_for(&key, 64 * 1024);
|
||||
put_object(writer, &bucket, &key, body.clone()).await?;
|
||||
let context = format!("worker={idx} round={round}");
|
||||
after_lock_contention(&format!("{context} PUT source"), || {
|
||||
writer
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(body.clone()))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
|
||||
let head = reader.head_object().bucket(&bucket).key(&key).send().await?;
|
||||
let head = after_lock_contention(&format!("{context} HEAD source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
if head.content_length() != Some(body.len() as i64) {
|
||||
return Err(format!("HEAD returned the wrong size for {key}: {head:?}").into());
|
||||
}
|
||||
assert_object_bytes(reader, &bucket, &key, &body).await?;
|
||||
assert_workload_bytes(reader, &bucket, &key, &body).await?;
|
||||
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_object_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
after_lock_contention(&format!("{context} COPY retained"), || {
|
||||
copier
|
||||
.copy_object()
|
||||
.bucket(&bucket)
|
||||
.key(©_key)
|
||||
.copy_source(format!("{bucket}/{key}"))
|
||||
.send()
|
||||
})
|
||||
.await??;
|
||||
assert_workload_bytes(writer, &bucket, ©_key, &body).await?;
|
||||
|
||||
writer.delete_object().bucket(&bucket).key(&key).send().await?;
|
||||
let missing = reader
|
||||
.head_object()
|
||||
.bucket(&bucket)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
after_lock_contention(&format!("{context} DELETE source"), || {
|
||||
writer.delete_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await??;
|
||||
let missing = after_lock_contention(&format!("{context} HEAD deleted source"), || {
|
||||
reader.head_object().bucket(&bucket).key(&key).send()
|
||||
})
|
||||
.await?
|
||||
.expect_err("deleted source key must not remain visible");
|
||||
if missing.raw_response().map(|response| response.status().as_u16()) != Some(404) {
|
||||
return Err(format!("deleted source {key} returned an unexpected result: {missing:?}").into());
|
||||
}
|
||||
@@ -82,14 +337,22 @@ async fn four_node_high_concurrency_mixed_workload_is_consistent_on_every_node()
|
||||
for handle in handles {
|
||||
inventory.extend(handle.await??);
|
||||
}
|
||||
assert_eq!(
|
||||
inventory.len(),
|
||||
worker_count * rounds,
|
||||
"every worker round must retain its acknowledged copy"
|
||||
);
|
||||
|
||||
let expected_keys: BTreeSet<_> = inventory.iter().map(|(key, _)| key.as_str()).collect();
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
let listed = client.list_objects_v2().bucket(&bucket).prefix("retained/").send().await?;
|
||||
let listed = after_lock_contention(&format!("node={node_idx} LIST retained"), || {
|
||||
client.list_objects_v2().bucket(&bucket).prefix("retained/").send()
|
||||
})
|
||||
.await??;
|
||||
let listed_keys: BTreeSet<_> = listed.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(listed_keys, expected_keys, "node {node_idx} returned a divergent retained-key listing");
|
||||
for (key, body) in &inventory {
|
||||
assert_object_bytes(client, &bucket, key, body)
|
||||
assert_workload_bytes(client, &bucket, key, body)
|
||||
.await
|
||||
.map_err(|error| format!("node {node_idx} failed to read {key}: {error}"))?;
|
||||
}
|
||||
|
||||
@@ -25,7 +25,8 @@
|
||||
|
||||
use super::harness::{
|
||||
DistCluster, DistLayout, TestResult, cluster_admin, cluster_admin_ok, decommission_status_json, get_object_bytes,
|
||||
payload_for, put_object, s3_probe_client, s3_probe_error_code, sha256_hex, unique_bucket, wait_until,
|
||||
payload_for, put_object, rebalance_complete, rebalance_status_json, s3_probe_client, s3_probe_error_code, sha256_hex,
|
||||
unique_bucket, wait_until,
|
||||
};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
|
||||
use crate::common::{FAST_DATA_USAGE_SCANNER_ENV, RustFSTestClusterEnvironment};
|
||||
@@ -47,6 +48,7 @@ use tokio::time::{Instant, sleep};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const MOVEMENT_BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
|
||||
pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
|
||||
pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
|
||||
pub(crate) const STAGE_BEFORE_SOURCE_CLEANUP: &str = "before_source_cleanup";
|
||||
pub(crate) const STAGE_AFTER_RESPONSIBILITY_PERSISTED: &str = "after_responsibility_persisted";
|
||||
@@ -1166,8 +1168,21 @@ pub(crate) async fn clear_decommission(cluster: &RustFSTestClusterEnvironment, p
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment) -> TestResult<String> {
|
||||
cluster_admin_ok(cluster, Method::POST, "/rustfs/admin/v3/rebalance/stop", None).await
|
||||
pub(crate) async fn stop_rebalance(cluster: &RustFSTestClusterEnvironment, expected_id: &str) -> TestResult<String> {
|
||||
let path = "/rustfs/admin/v3/rebalance/stop";
|
||||
let (status, response) = cluster_admin(cluster, Method::POST, path, None).await?;
|
||||
if status.is_success() {
|
||||
return Ok(response);
|
||||
}
|
||||
// The run can finish between the active observation and the stop request.
|
||||
// Only completion of that same run makes a not-started response acceptable.
|
||||
if status == StatusCode::NOT_FOUND
|
||||
&& response.contains("<Code>NoSuchResource</Code>")
|
||||
&& rebalance_complete(&rebalance_status_json(cluster).await?, expected_id)?
|
||||
{
|
||||
return Ok(response);
|
||||
}
|
||||
Err(format!("POST {path} failed: {status} {response}").into())
|
||||
}
|
||||
|
||||
pub(crate) fn decommission_pool_entry(status: &Value, pool_id: usize) -> TestResult<Value> {
|
||||
@@ -1585,6 +1600,129 @@ pub(crate) async fn assert_every_node_serves(dist: &DistCluster, bucket: &str, l
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const REBALANCE_NOT_STARTED: &str =
|
||||
"<Error><Code>NoSuchResource</Code><Message>pool rebalance is not started</Message></Error>";
|
||||
|
||||
async fn mock_rebalance_stop(
|
||||
stop_status: StatusCode,
|
||||
stop_body: &str,
|
||||
run_status: Value,
|
||||
) -> TestResult<(TestResult<String>, Vec<String>)> {
|
||||
use bytes::Bytes;
|
||||
use http_body_util::Full;
|
||||
use hyper::service::service_fn;
|
||||
use hyper_util::rt::TokioIo;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(1).await?;
|
||||
cluster.nodes[0].url = format!("http://{}", listener.local_addr()?);
|
||||
let stop_body = stop_body.to_string();
|
||||
let (requests_tx, mut requests_rx) = mpsc::unbounded_channel();
|
||||
let mut server = JoinSet::new();
|
||||
server.spawn(async move {
|
||||
loop {
|
||||
let (stream, _) = match listener.accept().await {
|
||||
Ok(connection) => connection,
|
||||
Err(error) => return Err::<(), Box<dyn std::error::Error + Send + Sync>>(error.into()),
|
||||
};
|
||||
let stop_body = stop_body.clone();
|
||||
let run_status = run_status.clone();
|
||||
let requests_tx = requests_tx.clone();
|
||||
let handler = service_fn(move |request: hyper::Request<hyper::body::Incoming>| {
|
||||
let route = format!("{} {}", request.method(), request.uri().path());
|
||||
requests_tx.send(route.clone()).expect("record mock admin request");
|
||||
let (status, body) = match route.as_str() {
|
||||
"POST /rustfs/admin/v3/rebalance/stop" => (stop_status, stop_body.clone()),
|
||||
"GET /rustfs/admin/v3/rebalance/status" => (StatusCode::OK, run_status.to_string()),
|
||||
_ => (StatusCode::BAD_REQUEST, "unexpected admin request".to_string()),
|
||||
};
|
||||
async move { http::Response::builder().status(status).body(Full::new(Bytes::from(body))) }
|
||||
});
|
||||
hyper::server::conn::http1::Builder::new()
|
||||
.keep_alive(false)
|
||||
.serve_connection(TokioIo::new(stream), handler)
|
||||
.await?;
|
||||
}
|
||||
});
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(5), stop_rebalance(&cluster, "run-1")).await?;
|
||||
server.abort_all();
|
||||
let requests = std::iter::from_fn(|| requests_rx.try_recv().ok()).collect();
|
||||
Ok((outcome, requests))
|
||||
}
|
||||
|
||||
fn completed_rebalance() -> Value {
|
||||
serde_json::json!({
|
||||
"id": "run-1",
|
||||
"pools": [
|
||||
{ "id": 0, "status": "Completed", "progress": { "versions": 113 } },
|
||||
{ "id": 1, "status": "None" }
|
||||
]
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stop_success_does_not_query_completion() -> TestResult {
|
||||
let (outcome, requests) = mock_rebalance_stop(StatusCode::OK, "", Value::Null).await?;
|
||||
assert_eq!(outcome?, "");
|
||||
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stop_accepts_completed_run_after_not_started_response() -> TestResult {
|
||||
let (outcome, requests) =
|
||||
mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, completed_rebalance()).await?;
|
||||
outcome?;
|
||||
assert_eq!(
|
||||
requests,
|
||||
[
|
||||
"POST /rustfs/admin/v3/rebalance/stop",
|
||||
"GET /rustfs/admin/v3/rebalance/status"
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_stop_rejects_unproven_completion_and_other_errors() -> TestResult {
|
||||
let complete = completed_rebalance();
|
||||
let mut invalid = vec![Value::Null];
|
||||
for (pointer, value) in [
|
||||
("/id", serde_json::json!("run-2")),
|
||||
("/pools/0/status", serde_json::json!("Started")),
|
||||
("/pools/0/status", serde_json::json!("Failed")),
|
||||
("/pools/0/status", serde_json::json!("Stopped")),
|
||||
("/pools/0/progress/versions", serde_json::json!(0)),
|
||||
] {
|
||||
let mut status = complete.clone();
|
||||
*status.pointer_mut(pointer).expect("completion fixture field") = value;
|
||||
invalid.push(status);
|
||||
}
|
||||
let mut pending = complete.clone();
|
||||
pending["stopPropagation"] = serde_json::json!({ "pendingTerminalReload": true });
|
||||
invalid.push(pending);
|
||||
let mut failed = complete.clone();
|
||||
failed["pools"][0]["lastError"] = serde_json::json!("migration failed");
|
||||
invalid.push(failed);
|
||||
for status in invalid {
|
||||
let (outcome, _) = mock_rebalance_stop(StatusCode::NOT_FOUND, REBALANCE_NOT_STARTED, status.clone()).await?;
|
||||
assert!(outcome.is_err(), "unproven completion accepted: {status}");
|
||||
}
|
||||
for (code, body) in [
|
||||
(StatusCode::FORBIDDEN, REBALANCE_NOT_STARTED),
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, REBALANCE_NOT_STARTED),
|
||||
(StatusCode::NOT_FOUND, "<Error><Code>NoSuchBucket</Code></Error>"),
|
||||
] {
|
||||
let (outcome, requests) = mock_rebalance_stop(code, body, complete.clone()).await?;
|
||||
assert!(outcome.is_err(), "unrelated error accepted: {code} {body}");
|
||||
assert_eq!(requests, ["POST /rustfs/admin/v3/rebalance/stop"]);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_ids_normalize_null_spellings() {
|
||||
for spelling in [None, Some(""), Some("null"), Some("00000000-0000-0000-0000-000000000000")] {
|
||||
|
||||
@@ -28,7 +28,7 @@ use super::harness::{
|
||||
wait_for_rebalance_active, wait_for_rebalance_complete,
|
||||
};
|
||||
use super::scanner_heal_topology_fixture::{
|
||||
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL,
|
||||
LiveWorkload, MOVEMENT_ACTIVE_TIMEOUT, MOVEMENT_COMPLETE_TIMEOUT, MovementOperation, SEED_POOL, STAGE_AFTER_ENTRY_ADMISSION,
|
||||
STAGE_AFTER_RESPONSIBILITY_PERSISTED, STAGE_BEFORE_SOURCE_CLEANUP, STAGE_BEFORE_TARGET_COMMIT, TopologyManifest, TopologyRun,
|
||||
assert_bucket_durability_strict, assert_every_node_serves, assert_probe_objects, assert_process_durability_strict,
|
||||
assert_seed_pool_census_unchanged, assert_single_pool_placement, cancel_decommission, capture_seed_pool_census,
|
||||
@@ -274,6 +274,7 @@ async fn wait_for_rebalance_stopped_or_completed(run: &TopologyRun, rebalance_id
|
||||
#[tokio::test]
|
||||
async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart() -> TestResult {
|
||||
init_logging();
|
||||
require_test_hooks();
|
||||
let mut run = TopologyRun::start("rebalance").await?;
|
||||
let manifest = seed_manifest(&run).await?;
|
||||
assert_bucket_durability_strict(&run.dist.cluster, &manifest.buckets()).await?;
|
||||
@@ -288,23 +289,52 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
|
||||
.map(|entry| (entry.key.clone(), entry.sha256.clone()))
|
||||
.collect();
|
||||
|
||||
let first_id = start_rebalance(&run.dist.cluster).await?;
|
||||
wait_for_rebalance_active(&run.dist.cluster, &first_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
let during_rebalance = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
|
||||
let workload = tokio::spawn(run_live_workload(LiveWorkload {
|
||||
clients: run.dist.clients()?,
|
||||
bucket: scratch.clone(),
|
||||
prefix: "live".to_string(),
|
||||
writers: LIVE_WRITERS,
|
||||
rounds: LIVE_ROUNDS,
|
||||
baseline_bucket: manifest.plain_bucket.clone(),
|
||||
baseline: baseline.clone(),
|
||||
pause: Duration::from_millis(100),
|
||||
}));
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
stop_rebalance(&run.dist.cluster).await?;
|
||||
let (first_outcome, first_status) = wait_for_rebalance_stopped_or_completed(&run, &first_id).await?;
|
||||
let ops = workload.await??;
|
||||
// Keep an admitted entry on the source until stop cancels admission. Start
|
||||
// propagation may await that entry, so observe and stop before awaiting ACK.
|
||||
let first_key = manifest.plain_key("external", 0)?;
|
||||
run.movement_barrier
|
||||
.arm_movement("rebalance", &manifest.plain_bucket, &first_key, STAGE_AFTER_ENTRY_ADMISSION)?;
|
||||
let observe_first = async {
|
||||
let (observed_id, reached) = tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, async {
|
||||
let reached = run.movement_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
let status = rebalance_status_json(&run.dist.cluster).await?;
|
||||
let observed_id = status
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("rebalance status omitted id")?;
|
||||
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
|
||||
})
|
||||
.await??;
|
||||
let probes = put_probe_objects(&run.dist, &scratch, "during-rebalance", 4).await?;
|
||||
let workload = run_live_workload(LiveWorkload {
|
||||
clients: run.dist.clients()?,
|
||||
bucket: scratch.clone(),
|
||||
prefix: "live".to_string(),
|
||||
writers: LIVE_WRITERS,
|
||||
rounds: LIVE_ROUNDS,
|
||||
baseline_bucket: manifest.plain_bucket.clone(),
|
||||
baseline: baseline.clone(),
|
||||
pause: Duration::from_millis(100),
|
||||
});
|
||||
let stop = async {
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
stop_rebalance(&run.dist.cluster, &observed_id).await?;
|
||||
let (outcome, status) = wait_for_rebalance_stopped_or_completed(&run, &observed_id).await?;
|
||||
if outcome != "stopped" {
|
||||
return Err("rebalance completed before stop; mid-run stop was not exercised".into());
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((outcome, status))
|
||||
};
|
||||
let (ops, (outcome, status)) = tokio::try_join!(workload, stop)?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id, reached, probes, ops, outcome, status))
|
||||
};
|
||||
let (first_id, (observed_id, first_reached, during_rebalance, ops, first_outcome, first_status)) =
|
||||
tokio::try_join!(start_rebalance(&run.dist.cluster), observe_first)?;
|
||||
if first_id != observed_id {
|
||||
return Err(format!("rebalance start acknowledged {first_id}, but the stopped run was {observed_id}").into());
|
||||
}
|
||||
run.movement_barrier.disarm()?;
|
||||
let live_state = expected_live_state(&ops);
|
||||
|
||||
for node_idx in 0..run.dist.cluster.nodes.len() {
|
||||
@@ -333,8 +363,40 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
|
||||
verify_from_every_node(&run.dist, &manifest, "after restart following the stop").await?;
|
||||
verify_live_state(&run.dist.client(0)?, &scratch, &live_state, "after restart following the stop").await?;
|
||||
|
||||
let second_id = start_rebalance(&run.dist.cluster).await?;
|
||||
wait_for_rebalance_active(&run.dist.cluster, &second_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
// A short second run can finish before the start response reaches us.
|
||||
// Hold one remaining source entry until its active state is observed.
|
||||
let mut second_key = None;
|
||||
for entry in manifest.plain_entries() {
|
||||
if pools_holding(&run.dist, &manifest.plain_bucket, &entry.key, None)?.contains(&SEED_POOL) {
|
||||
second_key = Some(entry.key.clone());
|
||||
break;
|
||||
}
|
||||
}
|
||||
let second_key = second_key.ok_or("no seed-pool object remains for the second rebalance")?;
|
||||
let second_barrier = run.movement_barrier.next_phase();
|
||||
second_barrier.arm_movement("rebalance", &manifest.plain_bucket, &second_key, STAGE_BEFORE_SOURCE_CLEANUP)?;
|
||||
let observe_second = async {
|
||||
let reached = second_barrier.wait_reached(MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
let status = rebalance_status_json(&run.dist.cluster).await?;
|
||||
let observed_id = status
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("rebalance status omitted id")?;
|
||||
if observed_id == first_id {
|
||||
return Err("the second rebalance reused the stopped run id".into());
|
||||
}
|
||||
wait_for_rebalance_active(&run.dist.cluster, observed_id, MOVEMENT_ACTIVE_TIMEOUT).await?;
|
||||
second_barrier.disarm()?;
|
||||
second_barrier.release_movement()?;
|
||||
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((observed_id.to_string(), reached))
|
||||
};
|
||||
// Worker propagation can hold the start response until movement drains.
|
||||
let (second_id, (observed_id, second_reached)) = tokio::try_join!(start_rebalance(&run.dist.cluster), async {
|
||||
tokio::time::timeout(MOVEMENT_ACTIVE_TIMEOUT, observe_second).await?
|
||||
})?;
|
||||
if second_id != observed_id {
|
||||
return Err(format!("rebalance start acknowledged {second_id}, but the active run was {observed_id}").into());
|
||||
}
|
||||
wait_for_rebalance_complete(&run.dist.cluster, &second_id, MOVEMENT_COMPLETE_TIMEOUT).await?;
|
||||
verify_from_every_node(&run.dist, &manifest, "after the second rebalance completed").await?;
|
||||
verify_live_state(&run.dist.client(3)?, &scratch, &live_state, "after the second rebalance completed").await?;
|
||||
@@ -351,8 +413,8 @@ async fn scanner_heal_topology_rebalance_survives_live_writes_stop_and_restart()
|
||||
&json!({
|
||||
"gate": "G-REB",
|
||||
"manifest": manifest_summary(&manifest),
|
||||
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop },
|
||||
"second_run": { "id": second_id, "versions_off_seed_pool": moved },
|
||||
"first_run": { "id": first_id, "outcome": first_outcome, "status": first_status, "versions_held_by_two_pools_after_stop": retained_after_stop, "barrier_object": first_key, "barrier": first_reached },
|
||||
"second_run": { "id": second_id, "versions_off_seed_pool": moved, "barrier_object": second_key, "barrier": second_reached },
|
||||
"live_ops": ops,
|
||||
"probe_placement_during_rebalance": during_placement,
|
||||
"process_durability_strict_nodes": strict_nodes,
|
||||
|
||||
@@ -33,7 +33,7 @@ mod tests {
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::io::{Read, Write};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
@@ -1654,6 +1654,89 @@ mod tests {
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
|
||||
async fn release_heal_commit_barrier_after_shutdown(
|
||||
log: &Path,
|
||||
log_offset: u64,
|
||||
barrier: &Path,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
// Observe within the existing graceful process-stop budget.
|
||||
timeout(Duration::from_secs(45), async {
|
||||
loop {
|
||||
let mut file = std::fs::File::open(log)?;
|
||||
file.seek(SeekFrom::Start(log_offset))?;
|
||||
let mut tail = String::new();
|
||||
file.read_to_string(&mut tail)?;
|
||||
if let Some(event) = tail
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.find(|event| {
|
||||
event["event"] == "server_shutdown_state"
|
||||
&& event["component"] == "main"
|
||||
&& event["subsystem"] == "startup"
|
||||
&& event["state"] == "stopping"
|
||||
})
|
||||
{
|
||||
// Local heal has stopped and HTTP shutdown has been signalled.
|
||||
// Let admitted peer RPCs finish; keeping this synthetic barrier
|
||||
// until process exit would make the fixture block its own drain.
|
||||
std::fs::remove_file(barrier)?;
|
||||
return Ok(event);
|
||||
}
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_commit_barrier_waits_for_new_shutdown_event() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let log = directory.path().join("server.log");
|
||||
let barrier = directory.path().join("barrier");
|
||||
let event = serde_json::json!({
|
||||
"event": "server_shutdown_state", "component": "main", "subsystem": "startup", "state": "stopping"
|
||||
});
|
||||
std::fs::write(&log, format!("{event}\n"))?;
|
||||
std::fs::write(&barrier, "bucket/prefix/")?;
|
||||
let log_offset = log.metadata()?.len();
|
||||
let mut writer = std::fs::OpenOptions::new().append(true).open(&log)?;
|
||||
writeln!(writer, "{}", serde_json::json!({"event": "shutdown_signal_received"}))?;
|
||||
writeln!(writer, "{}", serde_json::json!({"event": "server_shutdown_state", "state": "stopping"}))?;
|
||||
|
||||
let release = release_heal_commit_barrier_after_shutdown(&log, log_offset, &barrier);
|
||||
tokio::pin!(release);
|
||||
assert!(
|
||||
futures::poll!(release.as_mut()).is_pending(),
|
||||
"old or unrelated events must not release the barrier"
|
||||
);
|
||||
assert!(barrier.is_file(), "the partial-rebuild barrier must remain armed before shutdown");
|
||||
tokio::time::advance(Duration::from_secs(6)).await;
|
||||
assert!(
|
||||
futures::poll!(release.as_mut()).is_pending(),
|
||||
"background shutdown may take longer than five seconds within the process-stop budget"
|
||||
);
|
||||
writeln!(writer, "{event}")?;
|
||||
tokio::time::advance(Duration::from_millis(10)).await;
|
||||
assert_eq!(release.await?, event, "retain the actual new shutdown witness");
|
||||
assert!(!barrier.exists(), "shutdown must release the synthetic hold");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn heal_commit_barrier_missing_shutdown_fails_without_releasing() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let log = directory.path().join("server.log");
|
||||
let barrier = directory.path().join("barrier");
|
||||
std::fs::write(&log, "")?;
|
||||
std::fs::write(&barrier, "bucket/prefix/")?;
|
||||
let error = release_heal_commit_barrier_after_shutdown(&log, 0, &barrier)
|
||||
.await
|
||||
.expect_err("missing shutdown evidence must fail the fixture");
|
||||
assert!(error.is::<tokio::time::error::Elapsed>());
|
||||
assert!(barrier.is_file(), "a missing shutdown witness must not release repairs");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_case = match scenario {
|
||||
@@ -1791,6 +1874,7 @@ mod tests {
|
||||
let mut created_online_objects = Vec::with_capacity(online_object_count);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
let mut unclean_shutdown_marker_observed = None;
|
||||
let mut shutdown_barrier_release = None;
|
||||
let mut attempt_count = 0usize;
|
||||
let max_online_attempts = online_object_count.saturating_mul(topology.total_drives().max(1));
|
||||
while expected_manifests.len() < online_object_count {
|
||||
@@ -2373,7 +2457,21 @@ mod tests {
|
||||
| InterruptionScenario::BackgroundTargetRestartEc84MultiSet
|
||||
| InterruptionScenario::BackgroundCoordinatorRestart
|
||||
) {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
if interruption_node == 1
|
||||
&& let Some(barrier) = &commit_barrier
|
||||
{
|
||||
let target_log = PathBuf::from(format!("{log_dir}/node1.log"));
|
||||
let log_offset = target_log.metadata()?.len();
|
||||
let (stopped, released) = tokio::join!(
|
||||
cluster.stop_node_gracefully(interruption_node),
|
||||
release_heal_commit_barrier_after_shutdown(&target_log, log_offset, barrier),
|
||||
);
|
||||
// Await both branches even on error so the child is reaped.
|
||||
stopped?;
|
||||
shutdown_barrier_release = Some(released?);
|
||||
} else {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
}
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
}
|
||||
@@ -2415,7 +2513,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(barrier) = &commit_barrier {
|
||||
if shutdown_barrier_release.is_none()
|
||||
&& let Some(barrier) = &commit_barrier
|
||||
{
|
||||
std::fs::remove_file(barrier)?;
|
||||
}
|
||||
cluster.start_node_from_binary(interruption_node, &server_binary).await?;
|
||||
@@ -2746,6 +2846,7 @@ mod tests {
|
||||
"all_peers_bound_to_generation_window": true,
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"graceful_barrier_release": shutdown_barrier_release,
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
if let Some(outage_write) = outage_write_diagnostic {
|
||||
|
||||
@@ -283,6 +283,9 @@ mod ssec_copy_test;
|
||||
#[cfg(test)]
|
||||
mod multipart_storage_class_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod multipart_upload_id_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_class_capability_test;
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Multipart upload ID regression tests.
|
||||
//!
|
||||
//! Every operation that addresses an existing multipart upload must report an
|
||||
//! upload ID that does not name a live upload as `404 NoSuchUpload`, whether
|
||||
//! the ID is malformed or is a well-formed ID of an aborted upload.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
|
||||
|
||||
type TestResult = Result<(), Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const BUCKET: &str = "multipart-upload-id";
|
||||
const KEY: &str = "target.bin";
|
||||
const COPY_SOURCE_KEY: &str = "copy-source.bin";
|
||||
// RustFS upload IDs are URL-safe base64; '.' is outside that alphabet, so
|
||||
// the object layer rejects this ID as malformed instead of looking it up.
|
||||
const MALFORMED_UPLOAD_ID: &str = "malformed.upload.id";
|
||||
const PART_NUMBER: i32 = 1;
|
||||
const PART_BODY: &[u8] = b"multipart upload id regression part";
|
||||
const PART_ETAG: &str = "\"00000000000000000000000000000000\"";
|
||||
const NO_SUCH_UPLOAD_STATUS: u16 = 404;
|
||||
const NO_SUCH_UPLOAD_CODE: &str = "NoSuchUpload";
|
||||
const OBJECT_NOT_FOUND_STATUS: u16 = 404;
|
||||
|
||||
/// Describes how `result` differs from `404 NoSuchUpload`, or returns `None` when it matches.
|
||||
fn no_such_upload_mismatch<T, E>(operation: &str, result: Result<T, SdkError<E>>) -> Option<String>
|
||||
where
|
||||
E: ProvideErrorMetadata + std::fmt::Debug,
|
||||
{
|
||||
let Err(error) = result else {
|
||||
return Some(format!("{operation}: succeeded"));
|
||||
};
|
||||
let status = error.raw_response().map(|response| response.status().as_u16());
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
(status != Some(NO_SUCH_UPLOAD_STATUS) || code != Some(NO_SUCH_UPLOAD_CODE))
|
||||
.then(|| format!("{operation}: status {status:?}, code {code:?}: {error:?}"))
|
||||
}
|
||||
|
||||
async fn start_server_with_copy_source() -> Result<(RustFSTestEnvironment, Client), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(Vec::new()).await?;
|
||||
env.create_test_bucket(BUCKET).await?;
|
||||
let client = env.create_s3_client();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(COPY_SOURCE_KEY)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await?;
|
||||
Ok((env, client))
|
||||
}
|
||||
|
||||
/// Sends every operation that addresses an existing upload and asserts that all of them report `404 NoSuchUpload`.
|
||||
async fn assert_every_upload_operation_reports_no_such_upload(client: &Client, upload_id: &str) {
|
||||
let upload_part = client
|
||||
.upload_part()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.body(ByteStream::from_static(PART_BODY))
|
||||
.send()
|
||||
.await;
|
||||
let upload_part_copy = client
|
||||
.upload_part_copy()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.part_number(PART_NUMBER)
|
||||
.copy_source(format!("/{BUCKET}/{COPY_SOURCE_KEY}"))
|
||||
.send()
|
||||
.await;
|
||||
let list_parts = client.list_parts().bucket(BUCKET).key(KEY).upload_id(upload_id).send().await;
|
||||
let complete = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.multipart_upload(
|
||||
CompletedMultipartUpload::builder()
|
||||
.parts(CompletedPart::builder().part_number(PART_NUMBER).e_tag(PART_ETAG).build())
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await;
|
||||
let abort = client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await;
|
||||
|
||||
let mismatches: Vec<String> = [
|
||||
no_such_upload_mismatch("UploadPart", upload_part),
|
||||
no_such_upload_mismatch("UploadPartCopy", upload_part_copy),
|
||||
no_such_upload_mismatch("ListParts", list_parts),
|
||||
no_such_upload_mismatch("CompleteMultipartUpload", complete),
|
||||
no_such_upload_mismatch("AbortMultipartUpload", abort),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect();
|
||||
assert!(
|
||||
mismatches.is_empty(),
|
||||
"upload ID {upload_id:?}: operations not reporting {NO_SUCH_UPLOAD_STATUS} {NO_SUCH_UPLOAD_CODE}:\n{}",
|
||||
mismatches.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Asserts that the rejected operations left neither a multipart upload nor an object under `KEY`.
|
||||
async fn assert_no_upload_or_object_published(client: &Client) -> TestResult {
|
||||
let uploads = client.list_multipart_uploads().bucket(BUCKET).prefix(KEY).send().await?;
|
||||
assert!(uploads.uploads().is_empty(), "rejected operations must not leave an upload: {uploads:?}");
|
||||
let head = client.head_object().bucket(BUCKET).key(KEY).send().await;
|
||||
let status = head
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|error| error.raw_response())
|
||||
.map(|response| response.status().as_u16());
|
||||
assert_eq!(
|
||||
status,
|
||||
Some(OBJECT_NOT_FOUND_STATUS),
|
||||
"rejected operations must not publish {KEY}: {head:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, MALFORMED_UPLOAD_ID).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn aborted_upload_id_reports_no_such_upload_on_every_upload_operation() -> TestResult {
|
||||
let (_env, client) = start_server_with_copy_source().await?;
|
||||
let created = client.create_multipart_upload().bucket(BUCKET).key(KEY).send().await?;
|
||||
let upload_id = created.upload_id().ok_or("CreateMultipartUpload returned no upload ID")?;
|
||||
client
|
||||
.abort_multipart_upload()
|
||||
.bucket(BUCKET)
|
||||
.key(KEY)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_every_upload_operation_reports_no_such_upload(&client, upload_id).await;
|
||||
assert_no_upload_or_object_published(&client).await
|
||||
}
|
||||
}
|
||||
@@ -576,34 +576,6 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn reserve_object_metadata(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationRequest>,
|
||||
) -> Result<Response<SnapshotLeaseResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn activate_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationActivateRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn cancel_object_metadata_reservation(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::MetadataReservationCancelRequest>,
|
||||
) -> Result<Response<SnapshotLeaseMutationResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_reserved_data(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::ReservedRenameDataRequest>,
|
||||
) -> Result<Response<rustfs_protos::proto_gen::node_service::RenameDataResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn rename_data_with_pool_meta_cas(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::RenameDataRequest>,
|
||||
|
||||
@@ -31,8 +31,8 @@ use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, DeleteMarkerEntry, ObjectIdentifier, ObjectVersion,
|
||||
ServerSideEncryption, VersioningConfiguration,
|
||||
};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
@@ -3729,6 +3729,8 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let source_log = format!("{}/source.log", source_env.temp_dir);
|
||||
source_env.capture_log_path = Some(source_log.clone());
|
||||
let mut source_env_vars = replication_fast_env();
|
||||
source_env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_env_vars).await?;
|
||||
@@ -3791,19 +3793,66 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
"target did not preserve the source delete-marker version ID"
|
||||
);
|
||||
|
||||
source_client
|
||||
.delete_object()
|
||||
let wait_for_source_writeback = || async {
|
||||
let metrics_url = format!("{}/{source_bucket}?replication-metrics", source_env.url);
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let response = signed_request(
|
||||
http::Method::GET,
|
||||
&metrics_url,
|
||||
&source_env.access_key,
|
||||
&source_env.secret_key,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let metrics: serde_json::Value = response.error_for_status()?.json().await?;
|
||||
let queued = metrics["queued"]["curr"]["count"]
|
||||
.as_f64()
|
||||
.ok_or("missing replication queue count")?;
|
||||
if metrics["provider_available"] == true && queued == 0.0 {
|
||||
return Ok::<(), Box<dyn Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await?
|
||||
};
|
||||
wait_for_source_writeback().await?;
|
||||
|
||||
let purged = source_client
|
||||
.delete_objects()
|
||||
.bucket(source_bucket)
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(&purged_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.objects(
|
||||
ObjectIdentifier::builder()
|
||||
.key(object_key)
|
||||
.version_id(delete_marker_version_id)
|
||||
.build()?,
|
||||
)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(purged.errors().is_empty(), "batch version purge failed: {:?}", purged.errors());
|
||||
assert_eq!(purged.deleted().len(), 2, "both the data version and marker must be purged");
|
||||
assert_replication_converged(&source_client, source_bucket, &target_client, target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, target_bucket).await?;
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != purged_version_id),
|
||||
"target retained the explicitly purged object version"
|
||||
);
|
||||
assert!(
|
||||
target_state.iter().all(|entry| entry.version_id != delete_marker_version_id),
|
||||
"target retained the explicitly purged delete marker"
|
||||
);
|
||||
assert!(
|
||||
target_state
|
||||
.iter()
|
||||
@@ -3819,6 +3868,19 @@ async fn test_bucket_replication_converges_delete_marker_and_version_purge() ->
|
||||
.await?;
|
||||
assert_eq!(retained.body.collect().await?.into_bytes().as_ref(), b"versioned replication payload v2");
|
||||
|
||||
// Wait for the source workers to finish their final writeback, which runs
|
||||
// after the remote purge has already become visible (rustfs#9121).
|
||||
wait_for_source_writeback().await?;
|
||||
let logs = fs::read_to_string(source_log).await?;
|
||||
let writeback_failures: Vec<_> = logs
|
||||
.lines()
|
||||
.filter(|line| line.contains("apply_replication_delete_state"))
|
||||
.collect();
|
||||
assert!(
|
||||
writeback_failures.is_empty(),
|
||||
"a completed batch purge must not report a source writeback failure: {writeback_failures:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -94,3 +94,12 @@ pub(crate) mod scanner_peer_bucket {
|
||||
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
|
||||
pub(crate) use ecstore_rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
/// Storage handles used to seed a pre-validation bucket record while the fixture server is stopped.
|
||||
#[cfg(test)]
|
||||
pub(crate) mod bucket_tag_fixture {
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::load_bucket_metadata;
|
||||
pub(crate) use rustfs_ecstore::api::layout::{DisksLayout, EndpointServerPools};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::InstanceContext;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{ECStore, init_local_disks_with_instance_ctx};
|
||||
}
|
||||
|
||||
@@ -360,12 +360,26 @@ impl OpaMock {
|
||||
|
||||
pub(crate) async fn expect_bucket_tags(&mut self, action: &str, bucket: &str, department: Option<&str>) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, action, bucket, department);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn expect_versioned_tag_permission(&mut self, bucket: &str, object: &str, version: &str) -> TestResult {
|
||||
let payload = self.next_request().await?;
|
||||
Self::assert_bucket_tags(&payload, "s3:GetObjectVersionTagging", bucket, Some("finance"));
|
||||
assert_eq!(payload["input"]["resource"]["object"], object);
|
||||
assert_eq!(payload["input"]["context"]["conditions"]["versionid"], serde_json::json!([version]));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn assert_bucket_tags(payload: &Value, action: &str, bucket: &str, department: Option<&str>) {
|
||||
let input = &payload["input"];
|
||||
assert_eq!(input["identity"]["account"], "opabuckettags");
|
||||
assert_eq!(input["identity"]["is_owner"], false);
|
||||
assert_eq!(input["action"], action);
|
||||
assert_eq!(input["resource"]["bucket"], bucket);
|
||||
let expected = department.map(|value| serde_json::json!([value]));
|
||||
assert_eq!(input["context"]["conditions"].get("ExistingBucketTag/department"), expected.as_ref());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_validation(&mut self) -> TestResult {
|
||||
@@ -678,22 +692,61 @@ async fn test_list_buckets_opa_contract() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop and reap the directory's owning server before opening the fixture store.
|
||||
async fn seed_legacy_duplicate_bucket_tags(env: &mut RustFSTestEnvironment, bucket: &str) -> TestResult {
|
||||
use crate::storage_api::bucket_tag_fixture::{
|
||||
DisksLayout, ECStore, EndpointServerPools, InstanceContext, init_local_disks_with_instance_ctx, load_bucket_metadata,
|
||||
};
|
||||
|
||||
let server = env
|
||||
.process
|
||||
.as_mut()
|
||||
.ok_or("missing server process before offline fixture write")?;
|
||||
server.kill()?;
|
||||
server.wait()?;
|
||||
env.process = None;
|
||||
let layout = DisksLayout::from_volumes(std::slice::from_ref(&env.temp_dir))?;
|
||||
let (pools, _) = EndpointServerPools::create_server_endpoints(&env.address, &layout).await?;
|
||||
let context = std::sync::Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&context, pools.clone()).await?;
|
||||
let store = ECStore::new_with_instance_ctx(env.address.parse()?, pools, Default::default(), context.clone()).await?;
|
||||
let _cancel = context
|
||||
.background_cancel_token()
|
||||
.expect("store binds its shutdown token")
|
||||
.drop_guard();
|
||||
let mut metadata = load_bucket_metadata(store.clone(), bucket).await?;
|
||||
let incarnation = metadata.bucket_incarnation_id;
|
||||
let policy = metadata.policy_config_json.clone();
|
||||
assert!(!incarnation.is_nil());
|
||||
assert!(!policy.is_empty(), "the bucket-policy Allow must survive legacy seeding");
|
||||
// Encode the duplicate tag set with the same bucket record and erasure writer used by the store.
|
||||
let xml = b"<Tagging><TagSet><Tag><Key>department</Key><Value>finance</Value></Tag><Tag><Key>department</Key><Value>engineering</Value></Tag></TagSet></Tagging>";
|
||||
metadata.tagging_config_xml = xml.to_vec();
|
||||
metadata.save_with_store(store.clone()).await?;
|
||||
let persisted = load_bucket_metadata(store, bucket).await?;
|
||||
assert_eq!(persisted.bucket_incarnation_id, incarnation);
|
||||
assert_eq!(persisted.policy_config_json, policy);
|
||||
assert_eq!(persisted.tagging_config_xml.as_slice(), xml);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, VersioningConfiguration,
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Tag, Tagging, TaggingDirective, VersioningConfiguration,
|
||||
};
|
||||
|
||||
init_logging();
|
||||
let mut opa = OpaMock::start().await?;
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa.url.as_str()),
|
||||
let opa_url = opa.url.clone();
|
||||
let server_env = [
|
||||
("RUSTFS_POLICY_PLUGIN_URL", opa_url.as_str()),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", OPA_AUTH_TOKEN),
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
])
|
||||
.await?;
|
||||
];
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
let secret = uuid::Uuid::new_v4().to_string();
|
||||
create_user(&env, "opabuckettags", &secret).await?;
|
||||
let client = aws_sdk_s3::Client::from_conf(build_test_s3_config(&env.url, "opabuckettags", &secret, None, "e2e-bucket-tags"));
|
||||
@@ -733,7 +786,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
client
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
@@ -741,6 +794,8 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
assert_eq!(head.tag_count(), Some(1));
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
client.list_objects_v2().bucket(source).send().await?;
|
||||
opa.expect_bucket_tags("s3:ListBucket", source, Some("finance")).await?;
|
||||
client.list_object_versions().bucket(source).send().await?;
|
||||
@@ -750,28 +805,65 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
admin.create_bucket().bucket(destination).send().await?;
|
||||
set_department(&admin, destination, "engineering").await?;
|
||||
let copy_source = format!("{source}/report?versionId={version}");
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
let error = client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("source tags must not authorize a different destination");
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("denied COPY and REPLACE must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
}
|
||||
set_department(&admin, destination, "finance").await?;
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
for directive in [TaggingDirective::Copy, TaggingDirective::Replace] {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(destination)
|
||||
.key("copied")
|
||||
.copy_source(©_source)
|
||||
.set_tagging_directive((directive == TaggingDirective::Replace).then_some(TaggingDirective::Replace))
|
||||
.set_tagging((directive == TaggingDirective::Replace).then(|| "department=legal".to_owned()))
|
||||
.send()
|
||||
.await?;
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
// The access hook checks preliminary tags, then the copy use case checks
|
||||
// the exact source reader's tags before publishing the destination.
|
||||
for _ in 0..2 {
|
||||
if directive == TaggingDirective::Copy {
|
||||
opa.expect_versioned_tag_permission(source, "report", version).await?;
|
||||
}
|
||||
opa.expect_bucket_tags("s3:PutObject", destination, Some("finance")).await?;
|
||||
}
|
||||
let tags = admin.get_object_tagging().bucket(destination).key("copied").send().await?;
|
||||
assert_eq!(tags.tag_set().len(), 1);
|
||||
assert_eq!(tags.tag_set()[0].key(), "department");
|
||||
assert_eq!(
|
||||
tags.tag_set()[0].value(),
|
||||
if directive == TaggingDirective::Copy {
|
||||
"engineering"
|
||||
} else {
|
||||
"legal"
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
@@ -847,6 +939,14 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("engineering"))
|
||||
.await?;
|
||||
let missing = admin
|
||||
.head_object()
|
||||
.bucket(destination)
|
||||
.key("denied-source")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied source must leave no destination object");
|
||||
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
set_department(&client, source, "finance")
|
||||
.await
|
||||
.expect_err("requested tags must not authorize their own mutation");
|
||||
@@ -885,7 +985,7 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObject", destination, Some("engineering"))
|
||||
.await?;
|
||||
admin
|
||||
let invalid_write = admin
|
||||
.put_bucket_tagging()
|
||||
.bucket(destination)
|
||||
.tagging(
|
||||
@@ -895,7 +995,25 @@ async fn test_bucket_tags_opa_contract() -> TestResult {
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("new duplicate tags must be rejected before publication");
|
||||
assert_eq!(invalid_write.as_service_error().and_then(ProvideErrorMetadata::code), Some("InvalidTag"));
|
||||
assert_eq!(invalid_write.raw_response().map(|response| response.status().as_u16()), Some(400));
|
||||
|
||||
set_department(&admin, source, "finance").await?;
|
||||
seed_legacy_duplicate_bucket_tags(&mut env, destination).await?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&server_env).await?;
|
||||
// Prove this process can authorize the same caller and read a healthy bucket's persisted tags.
|
||||
let read = client
|
||||
.get_object()
|
||||
.bucket(source)
|
||||
.key("report")
|
||||
.version_id(version)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(read.body.collect().await?.into_bytes().as_ref(), b"report");
|
||||
opa.expect_bucket_tags("s3:GetObjectVersion", source, Some("finance")).await?;
|
||||
|
||||
let error = client
|
||||
.get_object()
|
||||
.bucket(destination)
|
||||
|
||||
@@ -48,6 +48,7 @@ const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
|
||||
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
|
||||
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
|
||||
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
|
||||
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
|
||||
const PLAIN_BUCKET: &str = "upgrade-plain-data";
|
||||
@@ -954,7 +955,8 @@ async fn exercise_mixed_cluster(
|
||||
phase: &str,
|
||||
current_node: usize,
|
||||
previous_node: usize,
|
||||
) -> TestResult {
|
||||
previous_has_delete_recovery: bool,
|
||||
) -> Result<Option<String>, BoxError> {
|
||||
let clients = cluster.create_all_clients()?;
|
||||
wait_for_upgrade_write_readiness(&clients, MIXED_BUCKET, phase, LISTING_CONVERGENCE_TIMEOUT).await?;
|
||||
wait_for_drive_readmission(&clients, MIXED_BUCKET, phase, DRIVE_READMISSION_TIMEOUT).await?;
|
||||
@@ -1009,6 +1011,30 @@ async fn exercise_mixed_cluster(
|
||||
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
|
||||
);
|
||||
|
||||
// 1.0.4 already implements the delete decision protocol. Use a separate
|
||||
// old-writer key so the original seed still spans the entire rollout.
|
||||
if previous_has_delete_recovery {
|
||||
let key = format!(".upgrade-delete/{phase}/previous-writer");
|
||||
previous_client
|
||||
.put_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.key(&key)
|
||||
.body(ByteStream::from_static(UPGRADE_READINESS_BODY))
|
||||
.send()
|
||||
.await?;
|
||||
for client in &clients {
|
||||
assert_eq!(read_object(client, MIXED_BUCKET, &key, None).await?.1, UPGRADE_READINESS_BODY);
|
||||
}
|
||||
upgrade_probe_client(current_client)
|
||||
.delete_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.key(&key)
|
||||
.send()
|
||||
.await?;
|
||||
assert_mixed_deletes_visible(&clients, std::slice::from_ref(&key)).await?;
|
||||
return Ok(Some(key));
|
||||
}
|
||||
|
||||
// Every online participant must understand the delete decision protocol.
|
||||
// An upgrade phase can serve reads/writes while intentionally refusing a
|
||||
// modern DELETE; rejecting it must leave the old writer's seed intact.
|
||||
@@ -1029,6 +1055,31 @@ async fn exercise_mixed_cluster(
|
||||
);
|
||||
}
|
||||
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn assert_mixed_deletes_visible(clients: &[Client], keys: &[String]) -> TestResult {
|
||||
for (node, client) in clients.iter().enumerate() {
|
||||
for key in keys {
|
||||
let error = upgrade_probe_client(client)
|
||||
.get_object()
|
||||
.bucket(MIXED_BUCKET)
|
||||
.key(key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a successful mixed-version DELETE must remain visible through every reader");
|
||||
assert_eq!(
|
||||
error.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"node {node}: {key}"
|
||||
);
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"node {node}: {key}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -2240,6 +2291,9 @@ async fn direct_upgrade_from_previous_release_preserves_object_contracts() -> Te
|
||||
async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts() -> TestResult {
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
// Bind the expected protocol to the verified release source, never to the
|
||||
// DELETE result itself. The older CI source pins retain their 503 contract.
|
||||
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
let current_binary = rustfs_binary_path();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
|
||||
@@ -2260,7 +2314,10 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
|
||||
cluster.stop_node(0)?;
|
||||
cluster.start_node_from_binary(0, ¤t_binary).await?;
|
||||
exercise_mixed_cluster(&cluster, "one-current-node", 0, 1).await?;
|
||||
let mut mixed_deleted_keys = Vec::new();
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
// One controlled out-of-policy attempt proves fail-closed behavior below
|
||||
// native write quorum; positive MPU writers remain paused while mixed.
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
@@ -2268,7 +2325,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
|
||||
cluster.stop_node(node_idx)?;
|
||||
cluster.start_node_from_binary(node_idx, ¤t_binary).await?;
|
||||
exercise_mixed_cluster(&cluster, phase, 0, 3).await?;
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
cluster.stop_node(3)?;
|
||||
@@ -2278,6 +2337,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
|
||||
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
|
||||
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
|
||||
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
|
||||
|
||||
for (node_idx, client) in clients.iter().enumerate() {
|
||||
assert_eq!(
|
||||
@@ -2326,6 +2386,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
cluster.start_with_binary(&previous_binary).await?;
|
||||
prepare_release_baseline(&mut cluster, &previous_binary, "rollback").await?;
|
||||
let old_clients = cluster.create_all_clients()?;
|
||||
assert_mixed_deletes_visible(&old_clients, &mixed_deleted_keys).await?;
|
||||
assert_resumed_multipart(&old_clients, &completed_pending).await?;
|
||||
for phase in ["homogeneous-previous", "homogeneous-current"] {
|
||||
verify_load_bodies(&old_clients, phase, "multipart").await?;
|
||||
@@ -2390,6 +2451,8 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"phases": phases,
|
||||
"objects_per_phase": objects_per_phase,
|
||||
"mixed_load": "plain-put",
|
||||
"mixed_delete_protocol_supported": previous_has_delete_recovery,
|
||||
"mixed_deleted_keys": mixed_deleted_keys,
|
||||
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
|
||||
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
|
||||
"assertions": [
|
||||
@@ -2422,7 +2485,11 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
|
||||
"the same legacy upload completes and replays once fully upgraded without adding a version",
|
||||
"modern DELETE fails closed and preserves every byte while old peers remain"
|
||||
if previous_has_delete_recovery {
|
||||
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
|
||||
} else {
|
||||
"modern DELETE fails closed and preserves every byte while old peers remain"
|
||||
}
|
||||
],
|
||||
}),
|
||||
)?;
|
||||
|
||||
@@ -31,6 +31,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips"]
|
||||
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
|
||||
# Sampled wall/CPU diagnostics only; does not change the read backend.
|
||||
diagnostic-local-batch = ["rustix/time"]
|
||||
|
||||
@@ -192,6 +192,7 @@ pub mod bucket {
|
||||
table_catalog_path_hash, unreadable_config_refusal,
|
||||
};
|
||||
pub use crate::bucket::metadata::{BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG};
|
||||
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, validate_tag_set_entries};
|
||||
}
|
||||
|
||||
pub mod config_parse_mode {
|
||||
@@ -331,8 +332,8 @@ pub mod bucket {
|
||||
pub mod utils {
|
||||
pub use crate::bucket::utils::{
|
||||
check_bucket_and_object_names, check_list_objs_args, check_object_name_for_length_and_slash,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, is_meta_bucketname, is_valid_object_prefix,
|
||||
serialize,
|
||||
check_valid_bucket_name_strict, deserialize, has_bad_path_component, has_xml_invalid_control_characters,
|
||||
is_meta_bucketname, is_valid_object_prefix, serialize,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -451,9 +452,15 @@ pub mod disk {
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::disk_store::wait_for_multipart_index_bootstrap;
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::disk::local::{local_durability_modes, set_local_rename_data_fail_after_metadata_commit};
|
||||
pub use crate::disk::multipart_index::{
|
||||
MULTIPART_INDEX_RPC_MAX_BYTES, MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES, MultipartIndexPage, MultipartIndexRpcReply,
|
||||
MultipartIndexRpcRequest,
|
||||
};
|
||||
pub use crate::disk::multipart_source::{
|
||||
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind,
|
||||
MultipartAbortRound, MultipartSourceOperation, MultipartSourcePhase, MultipartSourceRequest, MultipartSourceResponse,
|
||||
@@ -466,13 +473,15 @@ pub mod disk {
|
||||
pub use crate::disk::os::{LocalPublicationPause, LocalPublicationStage};
|
||||
pub use crate::disk::{
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration, CurrentDeleteTransition,
|
||||
BoundedDeleteRecoveryDiscovery, BoundedDeleteRecoveryInspection, BoundedDeleteRecoverySettlement,
|
||||
BoundedDeleteRecoverySettlementReply, CheckPartsResp, ConditionalFileUpdate, CurrentDeleteGeneration,
|
||||
CurrentDeleteTransition, DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES,
|
||||
DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction, DeleteRecoveryCursor, DeleteRecoveryDigest,
|
||||
DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation, DeleteRecoveryRpcReply, DeleteRecoveryState,
|
||||
Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions, FileReader, FileWriter,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup, PoolMetaNativeInspection,
|
||||
PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
|
||||
GuardedDeleteVersionsPayload, GuardedFileInfoVersions, HEAL_REPLACEMENT_RECOVERY_DIR, HEALING_MARKER_PATH,
|
||||
NsScannerOpenRequest, OldCurrentSize, POOL_META_NATIVE_CAS_METADATA_SUFFIX, PartTransactionAction, PoolMetaNativeBackup,
|
||||
PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
@@ -617,7 +626,7 @@ pub mod rebalance {
|
||||
pub mod rio {
|
||||
pub use crate::io_support::rio::{
|
||||
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
|
||||
boxed_reader, compression_metadata_value, wrap_reader,
|
||||
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -654,7 +663,9 @@ pub mod set_disk {
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
pub use crate::bucket::quota::reservation::fail_next_quota_ledger_save_for_test;
|
||||
pub use crate::bucket::quota::reservation::{
|
||||
fail_next_quota_ledger_save_for_test, initialize_quota_ledger_for_test, retry_quota_reconciliation_for_test,
|
||||
};
|
||||
pub use crate::set_disk::ConditionalReadBenchmarkMetadataGuard;
|
||||
pub use crate::set_disk::{MultipartCommitBarrier, MultipartCommitPause, PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
|
||||
|
||||
@@ -16,7 +16,8 @@ use crate::bucket::metadata::BucketMetadata;
|
||||
use crate::bucket::metadata_sys::get_bucket_targets_config;
|
||||
use crate::bucket::metadata_sys::get_replication_config;
|
||||
use crate::bucket::remote_s3_client::{
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, build_remote_s3_client,
|
||||
PathStyle, REPLICATION_TARGET_RETRY_POLICY, RemoteCredentials, RemoteS3EndpointSpec, RemoteS3RetryPolicy,
|
||||
UploadResponseDeadline, build_remote_s3_client,
|
||||
};
|
||||
use crate::bucket::replication::{ObjectLockIntegrity, object_lock_put_integrity, replication_etags_match};
|
||||
use crate::bucket::replication::{ReplicationStatusType, ReplicationTargetConfigBridge};
|
||||
@@ -119,6 +120,7 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
ca_cert_pem: (!target.ca_cert_pem.trim().is_empty()).then(|| target.ca_cert_pem.clone()),
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: Some(TargetRequestTimeouts::from_env().request),
|
||||
// Replication has no retry budget of its own on the request path,
|
||||
// so it keeps the SDK's standard three attempts; stating it here
|
||||
// pins the behaviour to this line instead of an SDK default.
|
||||
@@ -1191,6 +1193,7 @@ impl BucketTargetSys {
|
||||
health_check_duration: target.health_check_duration,
|
||||
replicate_sync: target.replication_sync,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::from_env(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1922,6 +1925,83 @@ impl std::fmt::Display for S3ClientError {
|
||||
}
|
||||
}
|
||||
|
||||
/// Replication request budgets, snapshotted when a target client is built.
|
||||
/// Streamed GET bodies are not subject to the upload-response budget.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct TargetRequestTimeouts {
|
||||
request: Duration,
|
||||
complete: Duration,
|
||||
minimum_transfer_rate: u64,
|
||||
}
|
||||
|
||||
impl Default for TargetRequestTimeouts {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
request: Duration::from_secs(60),
|
||||
complete: Duration::from_secs(15 * 60),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TargetRequestTimeouts {
|
||||
fn from_env() -> Self {
|
||||
let defaults = Self::default();
|
||||
let seconds = |name: &str, fallback: Duration| {
|
||||
let value = rustfs_utils::get_env_u64(name, fallback.as_secs());
|
||||
Duration::from_secs(if (1..=86400).contains(&value) {
|
||||
value
|
||||
} else {
|
||||
fallback.as_secs()
|
||||
})
|
||||
};
|
||||
let minimum_transfer_rate =
|
||||
rustfs_utils::get_env_u64("RUSTFS_REPLICATION_MIN_TRANSFER_RATE_BYTES_PER_SEC", defaults.minimum_transfer_rate);
|
||||
Self {
|
||||
request: seconds("RUSTFS_REPLICATION_REQUEST_TIMEOUT_SECS", defaults.request),
|
||||
complete: seconds("RUSTFS_REPLICATION_COMPLETE_TIMEOUT_SECS", defaults.complete),
|
||||
minimum_transfer_rate: if minimum_transfer_rate == 0 {
|
||||
defaults.minimum_transfer_rate
|
||||
} else {
|
||||
minimum_transfer_rate
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn upload_timeout(self, size: i64) -> Result<Duration, S3ClientError> {
|
||||
let size = u64::try_from(size).map_err(|_| S3ClientError::new("replication upload size must be non-negative"))?;
|
||||
// The SDK can make three attempts. Leave transmission headroom for
|
||||
// all three; the separate response deadline still bounds a trickling
|
||||
// target after each upload finishes.
|
||||
let attempts = match REPLICATION_TARGET_RETRY_POLICY {
|
||||
RemoteS3RetryPolicy::Disabled => 1,
|
||||
RemoteS3RetryPolicy::Standard { max_attempts } => u64::from(max_attempts.max(1)),
|
||||
};
|
||||
let seconds = size
|
||||
.div_ceil(self.minimum_transfer_rate)
|
||||
.checked_mul(attempts)
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
let timeout = self
|
||||
.request
|
||||
.checked_add(Duration::from_secs(seconds))
|
||||
.ok_or_else(|| S3ClientError::new("replication upload timeout exceeds supported duration"))?;
|
||||
if Instant::now().checked_add(timeout).is_none() {
|
||||
return Err(S3ClientError::new("replication upload timeout exceeds supported duration"));
|
||||
}
|
||||
Ok(timeout)
|
||||
}
|
||||
|
||||
fn upload_override(self, size: i64) -> Result<aws_sdk_s3::config::Builder, S3ClientError> {
|
||||
Ok(aws_sdk_s3::config::Builder::new()
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.upload_timeout(size)?)
|
||||
.build(),
|
||||
)
|
||||
.interceptor(UploadResponseDeadline(self.request)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TargetClient {
|
||||
pub endpoint: String,
|
||||
@@ -1935,6 +2015,7 @@ pub struct TargetClient {
|
||||
pub health_check_duration: Duration,
|
||||
pub replicate_sync: bool,
|
||||
pub client: Arc<S3Client>,
|
||||
pub timeouts: TargetRequestTimeouts,
|
||||
}
|
||||
|
||||
impl TargetClient {
|
||||
@@ -2360,12 +2441,11 @@ impl TargetClient {
|
||||
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
});
|
||||
let mut config = self.timeouts.upload_override(size)?;
|
||||
if forwards_source_checksum {
|
||||
operation = operation.config_override(
|
||||
aws_sdk_s3::config::Builder::new()
|
||||
.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired),
|
||||
);
|
||||
config = config.request_checksum_calculation(aws_sdk_s3::config::RequestChecksumCalculation::WhenRequired);
|
||||
}
|
||||
operation = operation.config_override(config);
|
||||
match operation.send().await {
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
@@ -2501,6 +2581,7 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(self.timeouts.upload_override(size)?)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2538,6 +2619,13 @@ impl TargetClient {
|
||||
}
|
||||
Result::<_, aws_smithy_types::error::operation::BuildError>::Ok(req)
|
||||
})
|
||||
.config_override(
|
||||
aws_sdk_s3::config::Builder::new().timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(self.timeouts.complete)
|
||||
.build(),
|
||||
),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
@@ -2966,11 +3054,241 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_headers,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TimedResponseConnector {
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: bytes::Bytes,
|
||||
frames: Arc<std::sync::atomic::AtomicUsize>,
|
||||
dropped: Arc<std::sync::atomic::AtomicUsize>,
|
||||
}
|
||||
|
||||
struct ResponseDrop(Arc<std::sync::atomic::AtomicUsize>);
|
||||
impl Drop for ResponseDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for TimedResponseConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let settings = self.clone();
|
||||
HttpConnectorFuture::new(async move {
|
||||
http_body_util::BodyExt::collect(request.into_body())
|
||||
.await
|
||||
.map_err(aws_smithy_runtime_api::client::result::ConnectorError::io)?;
|
||||
tokio::time::sleep(settings.upload_delay).await;
|
||||
let guard = ResponseDrop(Arc::clone(&settings.dropped));
|
||||
let stream = futures::stream::unfold(
|
||||
(settings.prefix_bytes, Some(settings.response), true, guard),
|
||||
move |(remaining, mut body, first, guard)| {
|
||||
let frames = Arc::clone(&settings.frames);
|
||||
async move {
|
||||
if remaining == 0 && body.is_none() {
|
||||
return None;
|
||||
}
|
||||
if !first {
|
||||
tokio::time::sleep(settings.response_delay).await;
|
||||
}
|
||||
let bytes = if remaining > 0 {
|
||||
bytes::Bytes::from_static(b" ")
|
||||
} else {
|
||||
body.take()?
|
||||
};
|
||||
frames.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
Some((
|
||||
Ok::<_, std::io::Error>(http_body::Frame::data(bytes)),
|
||||
(remaining.saturating_sub(1), body, false, guard),
|
||||
))
|
||||
}
|
||||
},
|
||||
)
|
||||
.fuse();
|
||||
let body = SdkBody::from_body_1_x(http_body_util::StreamBody::new(stream));
|
||||
let mut response =
|
||||
HttpResponse::new(aws_smithy_runtime_api::http::StatusCode::try_from(200u16).expect("status"), body);
|
||||
response.headers_mut().insert("etag", "\"test-etag\"");
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn timed_response_target(
|
||||
upload_delay: Duration,
|
||||
response_delay: Duration,
|
||||
prefix_bytes: usize,
|
||||
response: &'static [u8],
|
||||
) -> (TargetClient, Arc<std::sync::atomic::AtomicUsize>, Arc<std::sync::atomic::AtomicUsize>) {
|
||||
let frames = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let dropped = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let connector = SharedHttpConnector::new(TimedResponseConnector {
|
||||
upload_delay,
|
||||
response_delay,
|
||||
prefix_bytes,
|
||||
response: bytes::Bytes::from_static(response),
|
||||
frames: Arc::clone(&frames),
|
||||
dropped: Arc::clone(&dropped),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let (mut target, _) = header_recording_target_client(Vec::new());
|
||||
target.timeouts = TargetRequestTimeouts {
|
||||
request: Duration::from_secs(2),
|
||||
complete: Duration::from_secs(10),
|
||||
minimum_transfer_rate: 1024 * 1024,
|
||||
};
|
||||
target.client = Arc::new(S3Client::from_conf(
|
||||
target
|
||||
.client
|
||||
.config()
|
||||
.to_builder()
|
||||
.http_client(http_client)
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
|
||||
.timeout_config(
|
||||
aws_smithy_types::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(target.timeouts.request)
|
||||
.build(),
|
||||
)
|
||||
.build(),
|
||||
));
|
||||
(target, frames, dropped)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_upload_budgets_preserve_large_transfers_and_reject_invalid_sizes() {
|
||||
let policy = TargetRequestTimeouts::default();
|
||||
assert_eq!(policy.upload_timeout(0).expect("empty upload"), Duration::from_secs(60));
|
||||
assert_eq!(
|
||||
policy.upload_timeout(5 * 1024 * 1024 * 1024).expect("maximum single PUT"),
|
||||
Duration::from_secs(15420)
|
||||
);
|
||||
assert!(policy.upload_timeout(-1).is_err());
|
||||
let slow = TargetRequestTimeouts {
|
||||
minimum_transfer_rate: 1,
|
||||
..policy
|
||||
};
|
||||
assert!(slow.upload_timeout(i64::MAX).is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_upload_response_deadline_is_independent_of_size_and_checksum_override() {
|
||||
for operation in ["put", "forwarded-checksum", "part"] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::from_secs(4), Duration::from_millis(500), 100, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
let started = tokio::time::Instant::now();
|
||||
let error = if operation == "part" {
|
||||
target
|
||||
.put_object_part(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
1,
|
||||
size,
|
||||
ByteStream::from(body),
|
||||
&PutObjectPartOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
} else {
|
||||
let mut opts = PutObjectOptions::default();
|
||||
if operation == "forwarded-checksum" {
|
||||
opts.user_metadata.insert(
|
||||
"x-amz-checksum-sha256".to_owned(),
|
||||
"OoJ3yNhRwv3wwtZoGqEIPrPX9xwTnfLl+ka0wStN1g0=".to_owned(),
|
||||
);
|
||||
}
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &opts)
|
||||
.await
|
||||
.expect_err("response deadline")
|
||||
};
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_secs(6),
|
||||
"upload gets its full independent window: {error}"
|
||||
);
|
||||
assert!(
|
||||
started.elapsed() < target.timeouts.upload_timeout(size).expect("budget"),
|
||||
"response must end before the larger total deadline"
|
||||
);
|
||||
assert!(
|
||||
frames.load(std::sync::atomic::Ordering::SeqCst) >= 2,
|
||||
"trickle delivered progress before failure"
|
||||
);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1, "transport body released");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_slow_upload_preserves_transfer_window_and_streaming_get_body() {
|
||||
let (target, _, _) = timed_response_target(Duration::from_secs(4), Duration::from_millis(100), 5, b"");
|
||||
let body = vec![b'x'; 2 * 1024 * 1024];
|
||||
let size = i64::try_from(body.len()).expect("size");
|
||||
target
|
||||
.put_object("bucket", "key", size, ByteStream::from(body), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("slow upload, prompt response");
|
||||
let (target, _, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), 5, b"payload");
|
||||
let response = target
|
||||
.get_object("bucket", "key", None, None, None, HeaderMap::new())
|
||||
.await
|
||||
.expect("GET headers");
|
||||
let started = tokio::time::Instant::now();
|
||||
let body = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("streamed GET remains outside upload deadline")
|
||||
.into_bytes();
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert_eq!(&body[..], b" payload");
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_complete_processing_whitespace_has_a_separate_bounded_window() {
|
||||
const RESULT: &[u8] = b"<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><ETag>\"etag\"</ETag></CompleteMultipartUploadResult>";
|
||||
for prefix in [5, 100] {
|
||||
let (target, frames, dropped) = timed_response_target(Duration::ZERO, Duration::from_secs(1), prefix, RESULT);
|
||||
let started = tokio::time::Instant::now();
|
||||
let result = target
|
||||
.complete_multipart_upload(
|
||||
"bucket",
|
||||
"key",
|
||||
"upload",
|
||||
vec![CompletedPart::builder().part_number(1).e_tag("etag").build()],
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await;
|
||||
if prefix == 5 {
|
||||
assert!(result.is_ok(), "legal processing whitespace: {result:?}");
|
||||
} else {
|
||||
assert!(result.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(11));
|
||||
}
|
||||
assert!(started.elapsed() > target.timeouts.request);
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn replication_control_operation_deadline_covers_response_trickle() {
|
||||
let (target, frames, dropped) =
|
||||
timed_response_target(Duration::ZERO, Duration::from_millis(500), 100, b"<VersioningConfiguration/>");
|
||||
let started = tokio::time::Instant::now();
|
||||
assert!(target.get_bucket_versioning("bucket").await.is_err());
|
||||
assert!(started.elapsed() <= Duration::from_secs(3));
|
||||
assert!(frames.load(std::sync::atomic::Ordering::SeqCst) >= 2);
|
||||
assert_eq!(dropped.load(std::sync::atomic::Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
|
||||
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
|
||||
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
|
||||
@@ -3279,6 +3597,7 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
},
|
||||
request_uris,
|
||||
)
|
||||
@@ -3591,6 +3910,7 @@ mod tests {
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
timeouts: TargetRequestTimeouts::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -5487,6 +5487,14 @@ pub async fn put_restore_opts(
|
||||
if !oi.user_tags.is_empty() {
|
||||
meta.insert(AMZ_OBJECT_TAGGING.to_string(), (*oi.user_tags).clone());
|
||||
}
|
||||
if let Some(expires) = oi.expires {
|
||||
meta.insert(
|
||||
"expires".to_string(),
|
||||
expires
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.map_err(std::io::Error::other)?,
|
||||
);
|
||||
}
|
||||
let restore_expiry = lifecycle::expected_expiry_time(OffsetDateTime::now_utc(), rreq.days.unwrap_or(1));
|
||||
meta.insert(
|
||||
metadata_keys::RESTORE.to_string(),
|
||||
@@ -5506,7 +5514,6 @@ pub async fn put_restore_opts(
|
||||
// computed MD5 is not the object's public plaintext ETag.
|
||||
preserve_etag: oi.etag.clone(),
|
||||
shard_integrity_write_mode: Some(oi.shard_integrity_write_mode()),
|
||||
//expires: oi.expires,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
@@ -7144,6 +7151,25 @@ mod tests {
|
||||
0,
|
||||
"tier generation lease should release after EOF"
|
||||
);
|
||||
let unread = get_transitioned_object_reader_with_tier_manager(
|
||||
&object_info.bucket,
|
||||
&object_info.name,
|
||||
&None,
|
||||
&HeaderMap::new(),
|
||||
&object_info,
|
||||
&ObjectOptions::default(),
|
||||
&manager,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("open an unread transitioned response");
|
||||
assert_eq!(TierConfigMgr::active_operation_lease_count(&manager, &tier).await, 1);
|
||||
drop(unread);
|
||||
assert_eq!(
|
||||
TierConfigMgr::active_operation_lease_count(&manager, &tier).await,
|
||||
0,
|
||||
"canceling an unread response must release the tier lease"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
|
||||
@@ -256,6 +256,38 @@ pub const BUCKET_POLICY_CONFIG: &str = "policy.json";
|
||||
pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml";
|
||||
pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml";
|
||||
pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml";
|
||||
/// Shared tag-entry validation for object requests and persisted bucket tags.
|
||||
pub fn validate_tag_set_entries(tag_set: &[s3s::dto::Tag]) -> std::result::Result<(), &'static str> {
|
||||
fn is_valid_tag_text(value: &str) -> bool {
|
||||
static VALID_TAG_TEXT: std::sync::OnceLock<std::result::Result<regex::Regex, regex::Error>> = std::sync::OnceLock::new();
|
||||
VALID_TAG_TEXT
|
||||
.get_or_init(|| regex::Regex::new(r"^[\p{L}\p{N}\p{Z}+\-=._:/@]*$"))
|
||||
.as_ref()
|
||||
.is_ok_and(|pattern| pattern.is_match(value))
|
||||
}
|
||||
let mut keys = std::collections::HashSet::with_capacity(tag_set.len());
|
||||
for tag in tag_set {
|
||||
let key = tag
|
||||
.key
|
||||
.as_deref()
|
||||
.filter(|key| !key.is_empty())
|
||||
.ok_or("Tag key cannot be empty")?;
|
||||
if key.encode_utf16().count() > 128 || !is_valid_tag_text(key) {
|
||||
return Err("The TagKey you have provided is invalid");
|
||||
}
|
||||
let value = tag.value.as_deref().ok_or("Tag value cannot be null")?;
|
||||
if value.encode_utf16().count() > 256 || !is_valid_tag_text(value) {
|
||||
return Err("The TagValue you have provided is invalid");
|
||||
}
|
||||
if !keys.insert(key) {
|
||||
return Err("Cannot provide multiple Tags with the same key");
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub const MAX_BUCKET_TAGS: usize = 50;
|
||||
|
||||
pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml";
|
||||
pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json";
|
||||
pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml";
|
||||
@@ -990,6 +1022,20 @@ impl BucketMetadata {
|
||||
self.encryption_config_updated_at = updated;
|
||||
}
|
||||
BUCKET_TAGGING_CONFIG => {
|
||||
if !data.is_empty() {
|
||||
let tagging = deserialize::<Tagging>(&data).map_err(|_| {
|
||||
Error::InvalidArgument(self.name.clone(), String::new(), "Invalid bucket tagging XML".to_owned())
|
||||
})?;
|
||||
if tagging.tag_set.len() > MAX_BUCKET_TAGS {
|
||||
return Err(Error::InvalidArgument(
|
||||
self.name.clone(),
|
||||
String::new(),
|
||||
"Cannot have more than 50 tags per bucket".to_owned(),
|
||||
));
|
||||
}
|
||||
validate_tag_set_entries(&tagging.tag_set)
|
||||
.map_err(|message| Error::InvalidArgument(self.name.clone(), String::new(), message.to_owned()))?;
|
||||
}
|
||||
self.tagging_config_xml = data;
|
||||
// Drop the parsed form (like lifecycle above) so clearing the
|
||||
// payload can't leave stale parsed tags to be cached.
|
||||
@@ -1784,6 +1830,31 @@ mod test {
|
||||
/// backlog#2292: a replicated config is stamped with the source
|
||||
/// `updated_at` it was given, not the local clock, while the plain
|
||||
/// `update_config` entry keeps stamping the local clock.
|
||||
#[test]
|
||||
fn bucket_tag_updates_reject_unreadable_shapes_without_replacing_metadata() {
|
||||
let mut metadata = BucketMetadata::new("validated-tags");
|
||||
let valid = br#"<Tagging><TagSet><Tag><Key>team</Key><Value></Value></Tag></TagSet></Tagging>"#.to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, valid.clone())
|
||||
.expect("empty value is valid");
|
||||
let updated = metadata.tagging_config_updated_at;
|
||||
for invalid in [
|
||||
"<Tagging><TagSet><Tag><Key>team</Key></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key></Key><Value>x</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
"<Tagging><TagSet><Tag><Key>team</Key><Value>x</Value></Tag><Tag><Key>team</Key><Value>y</Value></Tag></TagSet></Tagging>".to_owned(),
|
||||
format!("<Tagging><TagSet>{}</TagSet></Tagging>", (0..51).map(|i| format!("<Tag><Key>k{i}</Key><Value>v</Value></Tag>")).collect::<String>()),
|
||||
] {
|
||||
assert!(metadata.update_config_at(BUCKET_TAGGING_CONFIG, invalid.into_bytes(), updated + time::Duration::hours(1)).is_err());
|
||||
assert_eq!(metadata.tagging_config_xml, valid);
|
||||
assert_eq!(metadata.tagging_config_updated_at, updated);
|
||||
}
|
||||
metadata.tagging_config_xml = b"legacy corrupt tags".to_vec();
|
||||
metadata
|
||||
.update_config(BUCKET_TAGGING_CONFIG, Vec::new())
|
||||
.expect("deletion repairs legacy corruption");
|
||||
assert!(metadata.tagging_config_xml.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_config_at_stamps_the_given_time_and_update_config_stamps_now() {
|
||||
let source_time = OffsetDateTime::now_utc() - time::Duration::hours(3);
|
||||
|
||||
@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -690,7 +691,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -3427,7 +3428,7 @@ mod tests {
|
||||
),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-rust</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
),
|
||||
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
|
||||
(
|
||||
@@ -4555,8 +4556,13 @@ mod tests {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("bucket volume should be created");
|
||||
}
|
||||
let mut bm = BucketMetadata::new(bucket);
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
if config_file == crate::bucket::metadata::BUCKET_TAGGING_CONFIG {
|
||||
// Model legacy persisted corruption without passing the write-time validator.
|
||||
bm.tagging_config_xml = raw.to_vec();
|
||||
} else {
|
||||
bm.update_config(config_file, raw.to_vec())
|
||||
.expect("raw config should be stored");
|
||||
}
|
||||
sys.persist_new_and_set(bm).await.expect("initial metadata should persist");
|
||||
}
|
||||
|
||||
@@ -5209,6 +5215,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
@@ -5682,7 +5743,8 @@ mod tests {
|
||||
// update only needs one interleaving to show up.
|
||||
const ROUNDS: usize = 8;
|
||||
for round in 0..ROUNDS {
|
||||
let tagging = format!("<Tagging><Round>{round}</Round></Tagging>").into_bytes();
|
||||
let tagging =
|
||||
format!("<Tagging><TagSet><Tag><Key>round</Key><Value>{round}</Value></Tag></TagSet></Tagging>").into_bytes();
|
||||
let policy = format!(r#"{{"Version":"2012-10-17","Round":{round}}}"#).into_bytes();
|
||||
|
||||
let start = Arc::new(tokio::sync::Barrier::new(2));
|
||||
@@ -5755,7 +5817,7 @@ mod tests {
|
||||
|
||||
let blocked = tokio::spawn({
|
||||
let node_b = node_b.clone();
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging/>".to_vec()).await }
|
||||
async move { update_with_sys(node_b, bucket, BUCKET_TAGGING_CONFIG, b"<Tagging><TagSet/></Tagging>".to_vec()).await }
|
||||
});
|
||||
|
||||
// Long enough for the write to have finished had it not waited: the
|
||||
|
||||
@@ -119,6 +119,21 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
// Exercise the real storage entry point before testing the new
|
||||
// reporting semantics: the same ledger must admit and refuse
|
||||
// these writes even when a scanner floor claims a million bytes.
|
||||
if protocol == 1 {
|
||||
let mut bootstrap = PutObjReader::from_vec(vec![0x5a; 512]);
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.put_object(&bucket, "bootstrap", &mut bootstrap, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::SlowDown)
|
||||
),
|
||||
"a missing authoritative ledger must not admit a write from scanner reporting"
|
||||
);
|
||||
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("bootstrap authoritative usage through the production worker");
|
||||
}
|
||||
let mut too_large = PutObjReader::from_vec(vec![0x5a; 8192]);
|
||||
assert!(
|
||||
matches!(
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -40,7 +40,9 @@ use aws_smithy_runtime_api::client::http::{
|
||||
HttpConnector as SmithyHttpConnector, HttpConnectorFuture, SharedHttpConnector, http_client_fn,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::interceptors::Intercept;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::BeforeTransmitInterceptorContextMut;
|
||||
use aws_smithy_runtime_api::client::interceptors::context::{
|
||||
BeforeDeserializationInterceptorContextMut, BeforeTransmitInterceptorContextMut,
|
||||
};
|
||||
use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};
|
||||
use aws_smithy_runtime_api::client::result::ConnectorError;
|
||||
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
|
||||
@@ -155,6 +157,8 @@ pub struct RemoteS3EndpointSpec {
|
||||
pub ca_cert_pem: Option<String>,
|
||||
pub connect_timeout: Option<Duration>,
|
||||
pub read_timeout: Option<Duration>,
|
||||
/// Entire SDK call, including non-streaming response bodies and retries.
|
||||
pub operation_timeout: Option<Duration>,
|
||||
/// How many wire requests one logical call may cost. Every consumer
|
||||
/// declares it; see [`RemoteS3RetryPolicy`].
|
||||
pub retry: RemoteS3RetryPolicy,
|
||||
@@ -252,6 +256,78 @@ pub(crate) fn remote_sdk_credentials(credentials: &RemoteCredentials, now: Syste
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
/// Only installed on non-streaming upload operations. The budget begins at
|
||||
/// response headers, independently of the potentially long request upload.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct UploadResponseDeadline(pub Duration);
|
||||
|
||||
impl Intercept for UploadResponseDeadline {
|
||||
fn name(&self) -> &'static str {
|
||||
"RustfsUploadResponseDeadline"
|
||||
}
|
||||
|
||||
fn modify_before_deserialization(
|
||||
&self,
|
||||
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), BoxError> {
|
||||
let response = context.response_mut();
|
||||
let inner = std::mem::replace(response.body_mut(), SdkBody::taken());
|
||||
*response.body_mut() = SdkBody::from_body_1_x(UploadResponseBody {
|
||||
inner: Some(inner),
|
||||
deadline: Box::pin(tokio::time::sleep(self.0)),
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
struct UploadResponseBody {
|
||||
inner: Option<SdkBody>,
|
||||
deadline: std::pin::Pin<Box<tokio::time::Sleep>>,
|
||||
}
|
||||
|
||||
impl http_body::Body for UploadResponseBody {
|
||||
type Data = bytes::Bytes;
|
||||
type Error = BoxError;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
||||
use std::future::Future as _;
|
||||
use std::task::Poll;
|
||||
if self.inner.is_none() {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
// The absolute response budget never renews on data, empty frames,
|
||||
// or trailers. A retry gets its own bounded response window.
|
||||
if self.deadline.as_mut().poll(cx).is_ready() {
|
||||
self.inner.take();
|
||||
return Poll::Ready(Some(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"replication response body deadline exceeded",
|
||||
)
|
||||
.into())));
|
||||
}
|
||||
let Some(inner) = self.inner.as_mut() else {
|
||||
return Poll::Ready(None);
|
||||
};
|
||||
let result = std::pin::Pin::new(inner).poll_frame(cx);
|
||||
if matches!(result, Poll::Ready(None) | Poll::Ready(Some(Err(_)))) {
|
||||
self.inner.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.inner.as_ref().is_none_or(http_body::Body::is_end_stream)
|
||||
}
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
self.inner.as_ref().map(http_body::Body::size_hint).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends a caller-identifying token to the SDK `User-Agent`. Runs after
|
||||
/// signing: SigV4 excludes `user-agent` from the canonical request, so the
|
||||
/// signature stays valid.
|
||||
@@ -309,7 +385,7 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
config_builder = config_builder.http_client(http_client);
|
||||
}
|
||||
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() {
|
||||
if spec.connect_timeout.is_some() || spec.read_timeout.is_some() || spec.operation_timeout.is_some() {
|
||||
let mut timeouts = TimeoutConfig::builder();
|
||||
if let Some(connect_timeout) = spec.connect_timeout {
|
||||
timeouts = timeouts.connect_timeout(connect_timeout);
|
||||
@@ -317,6 +393,9 @@ pub async fn build_remote_s3_config(spec: &RemoteS3EndpointSpec) -> Result<aws_s
|
||||
if let Some(read_timeout) = spec.read_timeout {
|
||||
timeouts = timeouts.read_timeout(read_timeout);
|
||||
}
|
||||
if let Some(operation_timeout) = spec.operation_timeout {
|
||||
timeouts = timeouts.operation_timeout(operation_timeout);
|
||||
}
|
||||
config_builder = config_builder.timeout_config(timeouts.build());
|
||||
}
|
||||
|
||||
@@ -674,6 +753,7 @@ mod tests {
|
||||
ca_cert_pem: None,
|
||||
connect_timeout: None,
|
||||
read_timeout: None,
|
||||
operation_timeout: None,
|
||||
retry: RemoteS3RetryPolicy::Disabled,
|
||||
user_agent_suffix: "",
|
||||
}
|
||||
@@ -1027,11 +1107,16 @@ mod tests {
|
||||
spec.user_agent_suffix = "RustFS-Test/0.0";
|
||||
spec.connect_timeout = Some(Duration::from_secs(5));
|
||||
spec.read_timeout = Some(Duration::from_secs(5));
|
||||
spec.operation_timeout = Some(Duration::from_secs(12));
|
||||
let config = build_remote_s3_config(&spec)
|
||||
.await
|
||||
.expect("spec should build")
|
||||
.http_client(http_client)
|
||||
.build();
|
||||
assert_eq!(
|
||||
config.timeout_config().and_then(TimeoutConfig::operation_timeout),
|
||||
Some(Duration::from_secs(12))
|
||||
);
|
||||
S3Client::from_conf(config)
|
||||
.head_bucket()
|
||||
.bucket("bucket")
|
||||
|
||||
@@ -33,6 +33,7 @@ pub(crate) struct ReplicationMetadataStore;
|
||||
impl ReplicationMetadataStore {
|
||||
pub(crate) const MRF_REPLICATION_FILE: &'static str = "config/replication/mrf.bin";
|
||||
pub(crate) const MRF_REPLICATION_RECOVERY_LOCK: &'static str = "config/replication/mrf.bin.recovery";
|
||||
pub(crate) const MRF_REPLICATION_REPLAY_STAMP: &'static str = "config/replication/mrf.bin.replay";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_FILE: &'static str = "config/replication/force-delete.bin";
|
||||
pub(crate) const FORCE_DELETE_REPLICATION_TRANSACTION_LOCK: &'static str = "config/replication/force-delete.bin.transaction";
|
||||
|
||||
@@ -116,6 +117,10 @@ mod tests {
|
||||
ReplicationMetadataStore::MRF_REPLICATION_RECOVERY_LOCK,
|
||||
"config/replication/mrf.bin.recovery"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
"config/replication/mrf.bin.replay"
|
||||
);
|
||||
assert_eq!(
|
||||
ReplicationMetadataStore::FORCE_DELETE_REPLICATION_FILE,
|
||||
"config/replication/force-delete.bin"
|
||||
|
||||
@@ -16,8 +16,8 @@ use super::replication_config_store::ReplicationConfigStore;
|
||||
use super::replication_error_boundary::{Error as EcstoreError, is_err_object_not_found, is_err_version_not_found};
|
||||
use super::replication_filemeta_boundary::{
|
||||
MrfOpKind, MrfReplicateEntry, REPLICATE_HEAL_DELETE, ReplicateDecision, ReplicateObjectInfo, ReplicatedTargetInfo,
|
||||
ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision, replicate_decision_for_admitted_targets,
|
||||
replication_statuses_map, version_purge_statuses_map,
|
||||
ReplicationGenerationSnapshot, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
|
||||
replicate_decision_for_admitted_targets, replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_REPLICATION_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION};
|
||||
@@ -86,6 +86,10 @@ const MRF_CORRUPT_FILE_PREFIX: &str = "config/replication/mrf.corrupt";
|
||||
const MRF_PENDING_CAP: usize = 200_000;
|
||||
const MRF_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(100);
|
||||
const MRF_RETRY_MAX_DELAY: Duration = Duration::from_secs(5);
|
||||
/// Interval between runtime replays of the MRF journal, cluster-wide. A pass
|
||||
/// re-attempts every retained entry, so a target that stays down costs one
|
||||
/// attempt per entry per interval.
|
||||
const MRF_REPLAY_INTERVAL: Duration = Duration::from_secs(300);
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DurableMrfBacklog {
|
||||
@@ -615,7 +619,7 @@ async fn read_mrf_entries<S: ReplicationObjectIO>(storage: Arc<S>) -> Result<Vec
|
||||
}
|
||||
|
||||
/// Acknowledge only the generation read by the recovery leader. Entries appended
|
||||
/// after that generation are retained as a suffix and replayed on the next startup.
|
||||
/// after that generation are retained as a suffix and replayed on the next pass.
|
||||
/// Lock order: recovery leader lock -> MRF journal object lock.
|
||||
async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
storage: Arc<S>,
|
||||
@@ -667,7 +671,7 @@ async fn acknowledge_mrf_recovery<S: ReplicationStorage>(
|
||||
Err(EcstoreError::PreconditionFailed)
|
||||
}
|
||||
|
||||
/// Acquires the MRF recovery leader lock for the startup replay.
|
||||
/// Acquires the MRF recovery leader lock for one journal replay pass.
|
||||
/// Returns `None` (after logging) when the lock cannot be created or another
|
||||
/// node is already processing the backlog.
|
||||
async fn acquire_mrf_recovery_guard<S: ReplicationStorage>(storage: &Arc<S>) -> Option<rustfs_lock::NamespaceLockGuard> {
|
||||
@@ -1018,7 +1022,7 @@ fn admitted_mrf_replicate_object(oi: ObjectInfo, entry: &MrfReplicateEntry, op_t
|
||||
}
|
||||
|
||||
/// Acknowledges the replayed MRF prefix and returns the retained backlog.
|
||||
/// On acknowledgement failure the backlog is preserved for the next startup and
|
||||
/// On acknowledgement failure the backlog is preserved for the next replay pass and
|
||||
/// re-read (falling back to the replayed snapshot) so the published summary stays accurate.
|
||||
async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
storage: &Arc<S>,
|
||||
@@ -1033,7 +1037,7 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next startup"
|
||||
"Failed to acknowledge the MRF recovery prefix; preserving it for the next replay pass"
|
||||
);
|
||||
match read_mrf_entries(storage.clone()).await {
|
||||
Ok(current) => current,
|
||||
@@ -1051,6 +1055,120 @@ async fn resolve_retained_mrf_entries<S: ReplicationStorage>(
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_millis_now() -> i64 {
|
||||
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000_000).unwrap_or(i64::MAX)
|
||||
}
|
||||
|
||||
/// Whether a periodic pass may run, given the start time of the last pass on
|
||||
/// any node. A stamp more than one interval ahead of this node's clock is skew
|
||||
/// or a bad value, so it does not suppress replay.
|
||||
fn mrf_replay_due(last_started_ms: Option<i64>, now_ms: i64, interval: Duration) -> bool {
|
||||
let Some(last_started_ms) = last_started_ms else {
|
||||
return true;
|
||||
};
|
||||
let interval_ms = i64::try_from(interval.as_millis()).unwrap_or(i64::MAX);
|
||||
now_ms.saturating_sub(last_started_ms) >= interval_ms || last_started_ms.saturating_sub(now_ms) >= interval_ms
|
||||
}
|
||||
|
||||
async fn read_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>) -> Option<i64> {
|
||||
let data = ReplicationConfigStore::read(storage.clone(), ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP)
|
||||
.await
|
||||
.ok()?;
|
||||
std::str::from_utf8(&data).ok()?.trim().parse().ok()
|
||||
}
|
||||
|
||||
async fn write_mrf_replay_stamp<S: ReplicationStorage>(storage: &Arc<S>, started_ms: i64) {
|
||||
if let Err(error) = ReplicationConfigStore::save(
|
||||
storage.clone(),
|
||||
ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP,
|
||||
started_ms.to_string().into_bytes(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
error = %error,
|
||||
"Failed to record the MRF replay start time"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One MRF journal replay pass. The recovery leader reads the journal,
|
||||
/// re-drives every entry, and acknowledges the replayed prefix; entries that
|
||||
/// could not be admitted, and entries appended meanwhile, stay for the next pass.
|
||||
///
|
||||
/// Every node runs the periodic loop, and the leader lock only serializes
|
||||
/// passes. With `min_interval`, a pass is skipped when a pass on any node
|
||||
/// started within that interval, read from the shared replay stamp, so a
|
||||
/// cluster replays once per interval rather than once per node. The startup
|
||||
/// pass passes `None` and always runs. A pass that finds entries records its
|
||||
/// start time. Returns whether the journal was read.
|
||||
/// Lock order: recovery leader lock -> replay stamp object lock.
|
||||
async fn replay_mrf_journal<S: ReplicationStorage>(storage: &Arc<S>, min_interval: Option<Duration>) -> bool {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(storage).await else {
|
||||
return false;
|
||||
};
|
||||
let started_ms = unix_millis_now();
|
||||
if let Some(interval) = min_interval
|
||||
&& !mrf_replay_due(read_mrf_replay_stamp(storage).await, started_ms, interval)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(storage).await else {
|
||||
return true;
|
||||
};
|
||||
if !entries.is_empty() {
|
||||
write_mrf_replay_stamp(storage, started_ms).await;
|
||||
}
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Re-runs [`replay_mrf_journal`] every `period`. The recovery leader lock keeps
|
||||
/// one pass at a time across the cluster, including the startup pass, and the
|
||||
/// shared replay stamp spaces passes from different nodes by `period`.
|
||||
fn spawn_mrf_replay_loop<S: ReplicationStorage>(storage: Arc<S>, period: Duration) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(period).await;
|
||||
replay_mrf_journal(&storage, Some(period)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("replication resync {active_resync_id} is already active for {bucket}/{arn}")]
|
||||
struct ResyncActiveConflictError {
|
||||
@@ -1171,6 +1289,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
// Start background tasks
|
||||
pool.start_mrf_persister().await;
|
||||
pool.start_mrf_processor().await;
|
||||
pool.start_mrf_replayer().await;
|
||||
pool.start_force_delete_processor().await;
|
||||
|
||||
pool
|
||||
@@ -1462,7 +1581,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues a replica task
|
||||
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
|
||||
// A version that is already queued or being uploaded is not driven a
|
||||
// A source generation already queued or being uploaded is not driven a
|
||||
// second time: the scanner heal pass sees it as PENDING until the
|
||||
// first upload lands and would otherwise re-queue it every cycle
|
||||
// (backlog#2362). The key is released when the worker finishes, or
|
||||
@@ -1639,55 +1758,26 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Starts the MRF processor — one-shot at startup.
|
||||
///
|
||||
/// Reads the on-disk MRF file, re-injects admitted entries as Heal operations, and
|
||||
/// rewrites any entries that could not be admitted for a later startup retry.
|
||||
/// Replays the on-disk MRF journal once; [`Self::start_mrf_replayer`] repeats
|
||||
/// the pass while the server runs.
|
||||
async fn start_mrf_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let Some(recovery_guard) = acquire_mrf_recovery_guard(&storage).await else {
|
||||
return;
|
||||
};
|
||||
|
||||
let Some(entries) = load_mrf_recovery_entries(&storage).await else {
|
||||
return;
|
||||
};
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&entries));
|
||||
|
||||
let total = entries.len();
|
||||
let mut queued_count = 0usize;
|
||||
let mut retry_entries = Vec::new();
|
||||
|
||||
for entry in entries.iter() {
|
||||
let Some(admission) = replay_mrf_entry(entry, &storage, &mut retry_entries).await else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if admission == ReplicationQueueAdmission::Missed {
|
||||
retry_entries.push(entry.clone());
|
||||
} else if admission == ReplicationQueueAdmission::Queued {
|
||||
queued_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let retained = resolve_retained_mrf_entries(&storage, &recovery_guard, &entries, &retry_entries).await;
|
||||
let retained_count = retained.len();
|
||||
set_durable_mrf_backlog_snapshot(durable_mrf_backlog_summary_from_entries(&retained));
|
||||
|
||||
if queued_count > 0 {
|
||||
info!(
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
recovered = queued_count,
|
||||
total,
|
||||
retained = retained_count,
|
||||
"Replayed MRF entries admitted for retry"
|
||||
);
|
||||
}
|
||||
replay_mrf_journal(&storage, None).await;
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
/// Starts the runtime MRF replayer — ongoing background task.
|
||||
///
|
||||
/// Entries reach the journal while the server runs (worker-queue overflow,
|
||||
/// delete-marker purge intents whose watch window drained with targets still
|
||||
/// dirty). Replaying only at startup left them waiting for a restart.
|
||||
async fn start_mrf_replayer(&self) {
|
||||
let handle = spawn_mrf_replay_loop(self.storage.clone(), MRF_REPLAY_INTERVAL);
|
||||
self.task_handles.lock().await.push(handle);
|
||||
}
|
||||
|
||||
async fn start_force_delete_processor(&self) {
|
||||
let storage = self.storage.clone();
|
||||
let handle =
|
||||
@@ -1755,7 +1845,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
/// writes them to the on-disk MRF file every flush interval (default 10s,
|
||||
/// overridable via `RUSTFS_REPL_MRF_FLUSH_INTERVAL_MS`) or when 1 000 new
|
||||
/// entries accumulate. Runtime writers append under the journal lock; only the
|
||||
/// startup recovery leader may remove an acknowledged prefix.
|
||||
/// recovery leader of a replay pass may remove an acknowledged prefix.
|
||||
async fn start_mrf_persister(&self) {
|
||||
let Some(mut rx) = self.mrf_save_rx.lock().await.take() else {
|
||||
return;
|
||||
@@ -2314,28 +2404,47 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Object versions currently queued or being uploaded, keyed by bucket,
|
||||
/// object name and version. `queue_replica_task` admits a version only once
|
||||
/// object name, version and source generation. `queue_replica_task` admits a generation only once
|
||||
/// while it is in flight; the scanner heal pass and MRF replays that arrive
|
||||
/// in the meantime are `Skipped` instead of driving a second complete upload
|
||||
/// (backlog#2362). Entries are removed when the worker finishes the task or
|
||||
/// when no worker accepted it.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct ReplicationInFlight {
|
||||
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
|
||||
keys: std::sync::Mutex<std::collections::HashSet<ReplicationInFlightKey>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
struct ReplicationInFlightKey {
|
||||
bucket: String,
|
||||
object: String,
|
||||
version_id: Option<uuid::Uuid>,
|
||||
generation: ReplicationGenerationSnapshot,
|
||||
}
|
||||
|
||||
impl From<&ReplicateObjectInfo> for ReplicationInFlightKey {
|
||||
fn from(info: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
bucket: info.bucket.clone(),
|
||||
object: info.name.clone(),
|
||||
version_id: info.version_id,
|
||||
generation: info.replication_generation.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ReplicationInFlight {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<ReplicationInFlightKey>> {
|
||||
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
/// Claim `ri`; `false` when the same version is already in flight.
|
||||
/// Claim `ri`; a metadata mutation is independent of its earlier generation.
|
||||
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
|
||||
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
|
||||
self.lock().insert(ReplicationInFlightKey::from(ri))
|
||||
}
|
||||
|
||||
fn finish(&self, ri: &ReplicateObjectInfo) {
|
||||
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
|
||||
self.lock().remove(&ReplicationInFlightKey::from(ri));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2348,26 +2457,21 @@ impl ReplicationInFlight {
|
||||
/// including when replication panics.
|
||||
struct ReplicationInFlightGuard {
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
key: ReplicateObjectInfo,
|
||||
key: ReplicationInFlightKey,
|
||||
}
|
||||
|
||||
impl ReplicationInFlightGuard {
|
||||
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
in_flight,
|
||||
key: ReplicateObjectInfo {
|
||||
bucket: ri.bucket.clone(),
|
||||
name: ri.name.clone(),
|
||||
version_id: ri.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
key: ReplicationInFlightKey::from(ri),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReplicationInFlightGuard {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.finish(&self.key);
|
||||
self.in_flight.lock().remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3421,6 +3525,8 @@ mod tests {
|
||||
allow_first_read: Notify,
|
||||
read_count: AtomicUsize,
|
||||
delete_storage_calls: AtomicUsize,
|
||||
delete_error: StdMutex<EcstoreError>,
|
||||
replay_stamp: StdMutex<Option<Vec<u8>>>,
|
||||
reported_size: StdMutex<Option<i64>>,
|
||||
stream_read_bytes: Arc<AtomicUsize>,
|
||||
write_count: AtomicUsize,
|
||||
@@ -3488,6 +3594,24 @@ mod tests {
|
||||
_h: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let stamp = self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned")
|
||||
.clone()
|
||||
.ok_or(EcstoreError::FileNotFound)?;
|
||||
return Ok(Self::GetObjectReader {
|
||||
object_info: ObjectInfo {
|
||||
size: i64::try_from(stamp.len()).expect("test stamp length should fit i64"),
|
||||
..Default::default()
|
||||
},
|
||||
stream: Box::new(Cursor::new(stamp)),
|
||||
buffered_body: None,
|
||||
body_source: Default::default(),
|
||||
});
|
||||
}
|
||||
if !object.ends_with("/.replication/resync.bin")
|
||||
&& !object.ends_with("config/replication/mrf.bin")
|
||||
&& !object.ends_with("config/replication/force-delete.bin")
|
||||
@@ -3551,6 +3675,16 @@ mod tests {
|
||||
let lock = self.new_ns_lock(bucket, object).await?;
|
||||
Some(lock.get_write_lock(Duration::from_secs(10)).await?)
|
||||
};
|
||||
if object.ends_with(ReplicationMetadataStore::MRF_REPLICATION_REPLAY_STAMP) {
|
||||
let mut stamp = Vec::new();
|
||||
data.stream.read_to_end(&mut stamp).await.map_err(EcstoreError::from)?;
|
||||
*self
|
||||
.shared
|
||||
.replay_stamp
|
||||
.lock()
|
||||
.expect("test replay stamp lock should not be poisoned") = Some(stamp);
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& let Some(replacement) = self
|
||||
.shared
|
||||
@@ -3672,7 +3806,12 @@ mod tests {
|
||||
_opts: Self::ObjectOptions,
|
||||
) -> Result<Self::ObjectInfo, Self::Error> {
|
||||
self.shared.delete_storage_calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(EcstoreError::NotImplemented)
|
||||
Err(self
|
||||
.shared
|
||||
.delete_error
|
||||
.lock()
|
||||
.expect("delete error lock should not be poisoned")
|
||||
.clone())
|
||||
}
|
||||
|
||||
async fn delete_objects(
|
||||
@@ -3929,6 +4068,48 @@ mod tests {
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_admits_metadata_changes_during_an_initial_upload() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, mut rx) = mpsc::channel(4);
|
||||
pool.workers.write().await.push(tx);
|
||||
let initial = ReplicateObjectInfo {
|
||||
bucket: "pending-tags-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let metadata = ReplicateObjectInfo {
|
||||
op_type: ReplicationType::Metadata,
|
||||
user_tags: String::new(),
|
||||
replication_generation: ReplicationGenerationSnapshot {
|
||||
mutation_id: Some(uuid::Uuid::new_v4().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..initial.clone()
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(initial.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
assert_eq!(pool.in_flight.len(), 2);
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &initial));
|
||||
assert_eq!(pool.in_flight.len(), 1);
|
||||
assert_eq!(pool.queue_replica_task(metadata.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
for expected in [&initial, &metadata] {
|
||||
let ReplicationOperation::Object(queued) = rx.recv().await.expect("queued operation") else {
|
||||
panic!("expected object replication");
|
||||
};
|
||||
assert_eq!(queued.replication_generation, expected.replication_generation);
|
||||
}
|
||||
drop(ReplicationInFlightGuard::new(pool.in_flight.clone(), &metadata));
|
||||
assert_eq!(pool.in_flight.len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
@@ -4171,6 +4352,8 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -4221,6 +4404,103 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_already_purged_source_completes_without_retry() {
|
||||
let arn = "arn:test";
|
||||
for delete_marker_version in [false, true] {
|
||||
for version_id in [Uuid::new_v4(), Uuid::nil()] {
|
||||
for error in [
|
||||
EcstoreError::VersionNotFound("source".to_string(), "purged".to_string(), version_id.to_string()),
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
EcstoreError::FileVersionNotFound,
|
||||
EcstoreError::FileNotFound,
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("purged-source", shared.clone()));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: (!delete_marker_version).then_some(version_id),
|
||||
delete_marker_version_id: delete_marker_version.then_some(version_id),
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
purge_targets: HashMap::from([(arn.to_string(), VersionPurgeStatusType::Complete)]),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
replicate_delete_with_outcome(delete, storage).await,
|
||||
"a confirmed target purge must complete when the source is already absent (marker={delete_marker_version}, version={version_id})"
|
||||
);
|
||||
assert_eq!(
|
||||
shared.delete_storage_calls.load(Ordering::SeqCst),
|
||||
2 + usize::from(!delete_marker_version && !version_id.is_nil()),
|
||||
"the worker must lock and attempt writeback, reading a non-null data version first"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_missing_source_does_not_hide_unfinished_replication() {
|
||||
let arn = "arn:test";
|
||||
for (version_purge, target_status, error) in [
|
||||
(
|
||||
true,
|
||||
VersionPurgeStatusType::Pending,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
(true, VersionPurgeStatusType::Complete, EcstoreError::ErasureWriteQuorum),
|
||||
(
|
||||
false,
|
||||
VersionPurgeStatusType::Empty,
|
||||
EcstoreError::ObjectNotFound("source".to_string(), "purged".to_string()),
|
||||
),
|
||||
] {
|
||||
let shared = empty_resync_shared_state();
|
||||
*shared.delete_error.lock().expect("delete error lock should not be poisoned") = error;
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("unfinished-delete", shared));
|
||||
let delete = DeletedObjectReplicationInfo {
|
||||
bucket: "source".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "purged".to_string(),
|
||||
version_id: version_purge.then(Uuid::new_v4),
|
||||
delete_marker: !version_purge,
|
||||
replication_state: Some(ReplicationState {
|
||||
replicate_decision_str: test_replicate_decision(&[arn]).to_string(),
|
||||
targets: if version_purge {
|
||||
HashMap::new()
|
||||
} else {
|
||||
HashMap::from([(arn.to_string(), ReplicationStatusType::Completed)])
|
||||
},
|
||||
purge_targets: if version_purge {
|
||||
HashMap::from([(arn.to_string(), target_status)])
|
||||
} else {
|
||||
HashMap::new()
|
||||
},
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
op_type: ReplicationType::Delete,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
!replicate_delete_with_outcome(delete, storage).await,
|
||||
"unfinished target work, source I/O failures, and marker creation must remain retryable"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_worker_preserves_persisted_admission_without_live_config() {
|
||||
let arn = "arn:test";
|
||||
@@ -5304,6 +5584,8 @@ mod tests {
|
||||
allow_first_read: Notify::new(),
|
||||
read_count: AtomicUsize::new(0),
|
||||
delete_storage_calls: AtomicUsize::new(0),
|
||||
delete_error: StdMutex::new(EcstoreError::NotImplemented),
|
||||
replay_stamp: StdMutex::new(None),
|
||||
reported_size: StdMutex::new(None),
|
||||
stream_read_bytes: Arc::new(AtomicUsize::new(0)),
|
||||
write_count: AtomicUsize::new(0),
|
||||
@@ -6268,6 +6550,108 @@ mod tests {
|
||||
assert_eq!(retained.target_arns, entry.target_arns);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_replay_loop_drains_entries_persisted_after_startup() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let storage = Arc::new(LoadResyncNodeStore::new("mrf-runtime-replay", shared.clone()));
|
||||
// A force-delete intent from a superseded generation replays as
|
||||
// Skipped, so a pass that reaches it acknowledges it off the journal.
|
||||
let stale_intent = |object: &str| MrfReplicateEntry {
|
||||
bucket: "mrf-runtime-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
let journal_drained = || async {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !shared.data.lock().expect("test data lock should not be poisoned").is_empty() {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.is_ok()
|
||||
};
|
||||
|
||||
let handle = spawn_mrf_replay_loop(storage, Duration::from_millis(20));
|
||||
for object in ["first", "second"] {
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[stale_intent(object)]).expect("MRF entry should encode");
|
||||
assert!(
|
||||
journal_drained().await,
|
||||
"an MRF entry persisted while the server runs must be replayed without a restart ({object})"
|
||||
);
|
||||
}
|
||||
handle.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_periodic_replay_runs_once_per_interval_across_nodes() {
|
||||
let shared = empty_resync_shared_state();
|
||||
let node_a = Arc::new(LoadResyncNodeStore::new("mrf-node-a", shared.clone()));
|
||||
let node_b = Arc::new(LoadResyncNodeStore::new("mrf-node-b", shared.clone()));
|
||||
let interval = Duration::from_secs(1);
|
||||
let persist_backlog = |object: &str| {
|
||||
let entry = MrfReplicateEntry {
|
||||
bucket: "mrf-node-replay".to_string(),
|
||||
object: object.to_string(),
|
||||
op: MrfOpKind::Delete,
|
||||
force_delete_id: Some(uuid::Uuid::new_v4()),
|
||||
force_delete_local_commit: false,
|
||||
target_arns: vec!["arn:rustfs:replication:target-a".to_string()],
|
||||
..Default::default()
|
||||
};
|
||||
*shared.data.lock().expect("test data lock should not be poisoned") =
|
||||
encode_mrf_file(&[entry]).expect("MRF entry should encode");
|
||||
};
|
||||
let journal_is_empty = || shared.data.lock().expect("test data lock should not be poisoned").is_empty();
|
||||
|
||||
persist_backlog("first");
|
||||
assert!(replay_mrf_journal(&node_a, Some(interval)).await, "node A's periodic pass should replay");
|
||||
assert!(journal_is_empty(), "node A's pass should acknowledge the replayed entry");
|
||||
|
||||
// Node B's timer fires within the same interval with a backlog present.
|
||||
persist_backlog("second");
|
||||
assert!(
|
||||
!replay_mrf_journal(&node_b, Some(interval)).await,
|
||||
"a node must not replay within one interval of another node's pass"
|
||||
);
|
||||
assert!(!journal_is_empty(), "the backlog should wait for the next interval");
|
||||
|
||||
assert!(
|
||||
replay_mrf_journal(&node_b, None).await,
|
||||
"the startup pass should replay regardless of the shared stamp"
|
||||
);
|
||||
assert!(journal_is_empty(), "the startup pass should acknowledge the replayed entry");
|
||||
|
||||
persist_backlog("third");
|
||||
tokio::time::sleep(interval).await;
|
||||
assert!(
|
||||
replay_mrf_journal(&node_a, Some(interval)).await,
|
||||
"a periodic pass should replay once the interval has elapsed"
|
||||
);
|
||||
assert!(journal_is_empty(), "the due pass should acknowledge the replayed entry");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_replay_due_tolerates_missing_and_skewed_stamps() {
|
||||
let interval = Duration::from_secs(300);
|
||||
let now = 1_000_000_000;
|
||||
assert!(mrf_replay_due(None, now, interval));
|
||||
assert!(!mrf_replay_due(Some(now - 299_999), now, interval));
|
||||
assert!(mrf_replay_due(Some(now - 300_000), now, interval));
|
||||
assert!(
|
||||
!mrf_replay_due(Some(now + 1_000), now, interval),
|
||||
"small clock skew should still space passes"
|
||||
);
|
||||
assert!(
|
||||
mrf_replay_due(Some(now + 300_000), now, interval),
|
||||
"a stamp an interval ahead must not suppress replay"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_entry_delete_marker_roundtrip() {
|
||||
let dm_vid = Uuid::new_v4();
|
||||
|
||||
@@ -2809,17 +2809,22 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
// A batch purge or another worker may have removed this version
|
||||
// already. Remote failures still control the final outcome below.
|
||||
let source_already_purged = is_version_purge && (is_err_object_not_found(&e) || is_err_version_not_found(&e));
|
||||
if !source_already_purged {
|
||||
error!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %dobj.target_arn,
|
||||
object = %dobj.delete_object.object_name,
|
||||
operation = "apply_replication_delete_state",
|
||||
error = %e,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
}
|
||||
send_event(EventArgs {
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
@@ -2832,7 +2837,7 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
},
|
||||
..Default::default()
|
||||
});
|
||||
false
|
||||
source_already_purged
|
||||
}
|
||||
};
|
||||
|
||||
@@ -3010,7 +3015,7 @@ const DELETE_MARKER_PURGE_WATCH_INTERVAL: TokioDuration = TokioDuration::from_se
|
||||
/// If the marker disappears (deleted before or while the replica landed),
|
||||
/// purge the replicated marker from the targets, retrying failed targets on
|
||||
/// later rounds. When the window drains with targets still dirty, persist the
|
||||
/// purge intent as a durable MRF entry so the next startup replays it through
|
||||
/// purge intent as a durable MRF entry so the next MRF replay pass re-drives it through
|
||||
/// `purge_stale_delete_marker_targets`.
|
||||
async fn watch_and_purge_source_delete_marker<S: ReplicationStorage>(
|
||||
bucket: String,
|
||||
@@ -3101,7 +3106,7 @@ async fn enqueue_delete_marker_purge_mrf(dobj: &DeletedObjectReplicationInfo, fa
|
||||
/// Heal callers retry for the full watch window because the startup MRF
|
||||
/// processor runs before bucket metadata (and thus target clients) finishes
|
||||
/// initializing — the first pass can see `target_client_missing` and a later
|
||||
/// round resolves the client; the replay loop is serial and startup-only, so
|
||||
/// round resolves the client; replay passes are serial background work, so
|
||||
/// blocking it for up to the window per dirty entry is acceptable. Live
|
||||
/// callers run on replication workers where a down target would pin a worker
|
||||
/// for the whole window, so they attempt once and lean on the durable intent
|
||||
@@ -6604,6 +6609,7 @@ mod tests {
|
||||
health_check_duration: std::time::Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(aws_sdk_s3::Client::from_conf(config)),
|
||||
timeouts: Default::default(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -259,6 +259,10 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
|
||||
pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) -> Result<()> {
|
||||
check_put_object_args(bucket, object)?;
|
||||
|
||||
if !data_movement && !is_meta_bucketname(bucket) && has_xml_invalid_control_characters(object) {
|
||||
return Err(StorageError::ObjectNameInvalid(bucket.to_owned(), object.to_owned()));
|
||||
}
|
||||
|
||||
if cfg!(target_os = "windows") && !data_movement && !is_meta_bucketname(bucket) && object.contains('\\') {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
@@ -270,6 +274,14 @@ pub fn check_new_object_args(bucket: &str, object: &str, data_movement: bool) ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// New public keys must remain representable in an unencoded XML listing.
|
||||
/// Existing keys still use the read/delete validators so they can be removed.
|
||||
pub fn has_xml_invalid_control_characters(object: &str) -> bool {
|
||||
object
|
||||
.chars()
|
||||
.any(|character| character < ' ' && !matches!(character, '\t' | '\n' | '\r'))
|
||||
}
|
||||
|
||||
pub fn check_copy_obj_args(bucket: &str, object: &str) -> Result<()> {
|
||||
check_bucket_and_object_names(bucket, object)
|
||||
}
|
||||
|
||||
@@ -34,7 +34,7 @@ use tokio::sync::Mutex as TokioMutex;
|
||||
use tokio::task::JoinSet;
|
||||
use tokio::time::timeout;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, warn};
|
||||
use tracing::{Instrument, debug, error, warn};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_METACACHE: &str = "metacache";
|
||||
@@ -224,6 +224,8 @@ pub struct ListPathRawOptions {
|
||||
pub recursive: bool,
|
||||
pub incl_deleted: bool,
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
pub hidden_entry_budget: usize,
|
||||
pub pending_prefix: Option<String>,
|
||||
pub filter_prefix: Option<String>,
|
||||
pub forward_to: Option<String>,
|
||||
pub min_disks: usize,
|
||||
@@ -260,6 +262,8 @@ impl Clone for ListPathRawOptions {
|
||||
recursive: self.recursive,
|
||||
incl_deleted: self.incl_deleted,
|
||||
skip_hidden_prefix_check: self.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: self.hidden_entry_budget,
|
||||
pending_prefix: self.pending_prefix.clone(),
|
||||
filter_prefix: self.filter_prefix.clone(),
|
||||
forward_to: self.forward_to.clone(),
|
||||
min_disks: self.min_disks,
|
||||
@@ -289,6 +293,8 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
recursive: opts.recursive,
|
||||
incl_deleted: opts.incl_deleted,
|
||||
skip_hidden_prefix_check: opts.skip_hidden_prefix_check,
|
||||
hidden_entry_budget: opts.hidden_entry_budget,
|
||||
pending_prefix: opts.pending_prefix.clone(),
|
||||
report_notfound: opts.report_not_found,
|
||||
filter_prefix: opts.filter_prefix.clone(),
|
||||
forward_to: opts.forward_to.clone(),
|
||||
@@ -301,13 +307,15 @@ fn walk_dir_options(opts: &ListPathRawOptions) -> WalkDirOptions {
|
||||
}
|
||||
|
||||
/// How a merged raw listing stream ended.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct ListPathRawOutcome {
|
||||
/// At least one producer stopped at its per-disk limit instead of at the
|
||||
/// end of its namespace, so the closed stream is not proof of exhaustion.
|
||||
/// Carried in-band by the metacache stream trailer, so remote producers
|
||||
/// report it as well.
|
||||
pub limit_reached: bool,
|
||||
/// Inclusive sorted boundary proved by every reader before a bounded stop.
|
||||
pub scan_cut: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn list_path_raw(rx: CancellationToken, opts: ListPathRawOptions) -> disk::error::Result<()> {
|
||||
@@ -363,10 +371,15 @@ async fn list_path_raw_inner(
|
||||
let producer_errs: Arc<[OnceLock<DiskError>]> = (0..opts.disks.len()).map(|_| OnceLock::new()).collect::<Vec<_>>().into();
|
||||
|
||||
let cancel_rx = CancellationToken::new();
|
||||
let walk_options = Arc::new(walk_dir_options(&opts));
|
||||
|
||||
for (disk_idx, disk) in opts.disks.iter().enumerate() {
|
||||
let opdisk = disk.clone();
|
||||
let opts_clone = opts.clone();
|
||||
let walk_options = walk_options.clone();
|
||||
#[cfg(test)]
|
||||
let test_reader_behavior = opts.test_reader_behaviors.get(disk_idx).cloned();
|
||||
#[cfg(test)]
|
||||
let test_disk_io_context_observed = opts.test_disk_io_context_observed.clone();
|
||||
let fallback_claim_tracker = fallback_claim_tracker.clone();
|
||||
let fds_clone = fds.clone();
|
||||
#[cfg(test)]
|
||||
@@ -376,17 +389,17 @@ async fn list_path_raw_inner(
|
||||
let disk_io_context_clone = disk_io_context.clone();
|
||||
let (rd, wr) = tokio::io::duplex(64 * 1024);
|
||||
readers.push(MetacacheReader::new(rd));
|
||||
jobs.spawn(with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let producer = with_optional_disk_io_context(disk_io_context_clone, async move {
|
||||
let mut wr = PublishedBytesWriter::new(wr);
|
||||
#[cfg(test)]
|
||||
if let Some(observer) = &opts_clone.test_disk_io_context_observed {
|
||||
if let Some(observer) = &test_disk_io_context_observed {
|
||||
observer.store(
|
||||
crate::disk::current_disk_io_context().is_some_and(|context| context.class == DiskIoClass::Background),
|
||||
std::sync::atomic::Ordering::Release,
|
||||
);
|
||||
}
|
||||
#[cfg(test)]
|
||||
let test_primary_error = if let Some(behavior) = opts_clone.test_reader_behaviors.get(disk_idx).cloned() {
|
||||
let test_primary_error = if let Some(behavior) = test_reader_behavior {
|
||||
match behavior {
|
||||
TestReaderBehavior::Eof => return Ok(()),
|
||||
TestReaderBehavior::Entries(entries) => {
|
||||
@@ -443,8 +456,6 @@ async fn list_path_raw_inner(
|
||||
None
|
||||
};
|
||||
|
||||
let wakl_opts = walk_dir_options(&opts_clone);
|
||||
|
||||
let mut need_fallback = false;
|
||||
let mut last_err = None;
|
||||
#[cfg(test)]
|
||||
@@ -459,7 +470,7 @@ async fn list_path_raw_inner(
|
||||
|
||||
if !need_fallback && let Some(disk) = opdisk {
|
||||
let primary_walk_started = std::time::Instant::now();
|
||||
match disk.walk_dir(wakl_opts, &mut wr).await {
|
||||
match disk.walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
Ok(_res) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_primary",
|
||||
@@ -483,8 +494,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -495,8 +506,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -598,8 +609,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing_for_path",
|
||||
error = ?err,
|
||||
@@ -610,8 +621,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_disk_missing",
|
||||
error = ?err,
|
||||
@@ -626,7 +637,7 @@ async fn list_path_raw_inner(
|
||||
}
|
||||
|
||||
let fallback_walk_started = std::time::Instant::now();
|
||||
match disk.as_ref().walk_dir(walk_dir_options(&opts_clone), &mut wr).await {
|
||||
match disk.as_ref().walk_dir((*walk_options).clone(), &mut wr).await {
|
||||
Ok(_r) => {
|
||||
rustfs_io_metrics::record_stage_duration(
|
||||
"metacache_walk_dir_fallback",
|
||||
@@ -652,8 +663,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_missing_path",
|
||||
error = ?err,
|
||||
@@ -664,8 +675,8 @@ async fn list_path_raw_inner(
|
||||
event = EVENT_METACACHE_LISTING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_METACACHE,
|
||||
bucket = %opts_clone.bucket,
|
||||
path = %opts_clone.path,
|
||||
bucket = %walk_options.bucket,
|
||||
path = %walk_options.base_dir,
|
||||
disk_index = disk_idx,
|
||||
state = "fallback_walk_dir_failed",
|
||||
error = ?err,
|
||||
@@ -687,12 +698,13 @@ async fn list_path_raw_inner(
|
||||
|
||||
// warn!("list_path_raw: while need_fallback done");
|
||||
Ok(())
|
||||
}));
|
||||
});
|
||||
jobs.spawn(producer.in_current_span());
|
||||
}
|
||||
|
||||
let revjob_rx = rx.clone();
|
||||
let mut revjobs = JoinSet::new();
|
||||
revjobs.spawn(async move {
|
||||
let merge = async move {
|
||||
// Consumer-side peek timeout, and a caveat worth understanding
|
||||
// (rustfs/backlog#1217).
|
||||
//
|
||||
@@ -722,6 +734,8 @@ async fn list_path_raw_inner(
|
||||
// Sticky across rounds: once any reader's trailer says its producer
|
||||
// stopped at the per-disk limit, the merged stream may hide more keys.
|
||||
let mut limit_reached = false;
|
||||
let mut scan_cut = None;
|
||||
let mut last_scanned_key: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut current = MetaCacheEntry::default();
|
||||
@@ -733,7 +747,7 @@ async fn list_path_raw_inner(
|
||||
// );
|
||||
|
||||
if revjob_rx.is_cancelled() {
|
||||
return Ok(ListPathRawOutcome { limit_reached });
|
||||
return Ok(ListPathRawOutcome { limit_reached, scan_cut });
|
||||
}
|
||||
|
||||
let mut top_entries: Vec<Option<MetaCacheEntry>> = vec![None; readers.len()];
|
||||
@@ -793,7 +807,12 @@ async fn list_path_raw_inner(
|
||||
continue;
|
||||
}
|
||||
// eof
|
||||
limit_reached |= r.limit_reached() == Some(true);
|
||||
if r.limit_reached() == Some(true) {
|
||||
limit_reached = true;
|
||||
scan_cut = Some(last_scanned_key.clone().ok_or_else(|| {
|
||||
DiskError::other("bounded listing reader ended without a scan boundary")
|
||||
})?);
|
||||
}
|
||||
at_eof += 1;
|
||||
// warn!("list_path_raw: peek eof, disk: {}", i);
|
||||
continue;
|
||||
@@ -867,7 +886,7 @@ async fn list_path_raw_inner(
|
||||
*r = MetacacheReader::new(detached_rd);
|
||||
continue;
|
||||
}
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
PeekOutcome::Cancelled => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
}
|
||||
};
|
||||
|
||||
@@ -926,7 +945,7 @@ async fn list_path_raw_inner(
|
||||
if let Some(finished_fn) = opts.finished.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -976,6 +995,14 @@ async fn list_path_raw_inner(
|
||||
return Err(classify_listing_quorum_failure(&failures));
|
||||
}
|
||||
|
||||
// The next head from a bounded reader is unknown. The previous
|
||||
// globally consumed group is a safe cut; consuming another reader's
|
||||
// later head would turn this absence of evidence into a skipped key.
|
||||
// Check quorum failures above before accepting the bounded result.
|
||||
if limit_reached {
|
||||
break;
|
||||
}
|
||||
|
||||
// Break if all at EOF or error.
|
||||
if at_eof + has_err == readers.len() {
|
||||
if has_err > 0
|
||||
@@ -983,7 +1010,7 @@ async fn list_path_raw_inner(
|
||||
{
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = finished_fn(&errs) => {}
|
||||
}
|
||||
}
|
||||
@@ -994,6 +1021,7 @@ async fn list_path_raw_inner(
|
||||
break;
|
||||
}
|
||||
|
||||
last_scanned_key = Some(current.name.clone());
|
||||
if agree == readers.len() && !opts.preserve_replica_metadata {
|
||||
for r in readers.iter_mut() {
|
||||
if !r.skip_current() {
|
||||
@@ -1005,7 +1033,7 @@ async fn list_path_raw_inner(
|
||||
// warn!("list_path_raw: agreed_fn start, current: {:?}", ¤t.name);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
result = agreed_fn(current) => result?
|
||||
}
|
||||
// warn!("list_path_raw: agreed_fn done");
|
||||
@@ -1029,13 +1057,14 @@ async fn list_path_raw_inner(
|
||||
if let Some(partial_fn) = opts.partial.as_ref() {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached }),
|
||||
_ = revjob_rx.cancelled() => return Ok(ListPathRawOutcome { limit_reached, scan_cut }),
|
||||
_ = partial_fn(MetaCacheEntries(top_entries), &errs) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ListPathRawOutcome { limit_reached })
|
||||
});
|
||||
Ok(ListPathRawOutcome { limit_reached, scan_cut })
|
||||
};
|
||||
revjobs.spawn(merge.in_current_span());
|
||||
|
||||
let merge_started = std::time::Instant::now();
|
||||
let Some(revjob) = revjobs.join_next().await else {
|
||||
@@ -1199,6 +1228,24 @@ mod tests {
|
||||
assert!(!options.skip_total_timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cloned_reader_options_preserve_hidden_scan_budget_and_pending_prefix() {
|
||||
let request = ListPathRawOptions {
|
||||
bucket: "versioned".to_owned(),
|
||||
hidden_entry_budget: 100,
|
||||
pending_prefix: Some("pending/".to_owned()),
|
||||
forward_to: Some("pending/0099".to_owned()),
|
||||
..Default::default()
|
||||
};
|
||||
// Both walks are built once from the shared options; the per-job clone
|
||||
// must carry every field that the walk derives from the request.
|
||||
let job = request.clone();
|
||||
let walk = walk_dir_options(&job);
|
||||
assert_eq!(walk.hidden_entry_budget, 100);
|
||||
assert_eq!(walk.pending_prefix.as_deref(), request.pending_prefix.as_deref());
|
||||
assert_eq!(walk.forward_to.as_deref(), Some("pending/0099"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_empty_disks_returns_read_quorum() {
|
||||
let err = list_path_raw(CancellationToken::new(), ListPathRawOptions::default())
|
||||
@@ -1249,6 +1296,163 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn list_path_raw_fanout_retains_only_linear_disk_references() {
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let baseline = Arc::strong_count(&disk);
|
||||
|
||||
for primary_count in [1, 4, 16] {
|
||||
for fallback_count in [0, 2] {
|
||||
let entered: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let stopped: Vec<_> = (0..primary_count).map(|_| Arc::new(tokio::sync::Notify::new())).collect();
|
||||
let behaviors = entered
|
||||
.iter()
|
||||
.zip(&stopped)
|
||||
.map(|(entered, stopped)| TestReaderBehavior::StallWithSignals {
|
||||
entered: entered.clone(),
|
||||
stopped: stopped.clone(),
|
||||
})
|
||||
.collect();
|
||||
let cancel = CancellationToken::new();
|
||||
let listing = tokio::spawn(list_path_raw(
|
||||
cancel.clone(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk.clone()); primary_count],
|
||||
fallback_disks: vec![Some(disk.clone()); fallback_count],
|
||||
min_disks: primary_count,
|
||||
test_reader_behaviors: behaviors,
|
||||
peek_timeout: Some(Duration::from_secs(60)),
|
||||
..Default::default()
|
||||
},
|
||||
));
|
||||
for signal in &entered {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("every producer must enter the real fanout before counting references");
|
||||
}
|
||||
|
||||
// The merge owns the configured primary/fallback slots, each producer
|
||||
// owns its primary, and the shared fallback queue owns each fallback.
|
||||
let expected = baseline + 2 * (primary_count + fallback_count);
|
||||
let retained = Arc::strong_count(&disk);
|
||||
cancel.cancel();
|
||||
timeout(Duration::from_secs(2), listing)
|
||||
.await
|
||||
.expect("cancellation must stop the listing")
|
||||
.expect("listing task must join")
|
||||
.expect("external cancellation must finish cleanly");
|
||||
for signal in &stopped {
|
||||
timeout(Duration::from_secs(2), signal.notified())
|
||||
.await
|
||||
.expect("cancellation must drop every producer");
|
||||
}
|
||||
assert_eq!(Arc::strong_count(&disk), baseline, "cancellation must release every fanout reference");
|
||||
assert_eq!(
|
||||
retained, expected,
|
||||
"fanout must retain linear references for {primary_count} primary and {fallback_count} fallback slots"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn list_path_raw_producer_and_merge_preserve_request_span() {
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt};
|
||||
|
||||
struct ProducerSpan(Arc<AtomicBool>);
|
||||
#[derive(Default)]
|
||||
struct StateField(bool);
|
||||
impl tracing::field::Visit for StateField {
|
||||
fn record_debug(&mut self, _: &tracing::field::Field, _: &dyn std::fmt::Debug) {}
|
||||
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
self.0 |= field.name() == "state" && value == "walk_dir_missing_path";
|
||||
}
|
||||
}
|
||||
impl<S> Layer<S> for ProducerSpan
|
||||
where
|
||||
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
|
||||
{
|
||||
fn on_event(&self, event: &tracing::Event<'_>, context: tracing_subscriber::layer::Context<'_, S>) {
|
||||
let mut state = StateField::default();
|
||||
event.record(&mut state);
|
||||
if state.0
|
||||
&& context
|
||||
.lookup_current()
|
||||
.is_some_and(|span| span.name() == "flink_list_request")
|
||||
{
|
||||
self.0.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The subscriber is thread-local; keep the spawned tasks on this runtime thread.
|
||||
let producer_observed = Arc::new(AtomicBool::new(false));
|
||||
let subscriber = tracing_subscriber::registry().with(ProducerSpan(producer_observed.clone()));
|
||||
let _subscriber = tracing::subscriber::set_default(subscriber);
|
||||
let request = tracing::info_span!("flink_list_request", request_id = "list-span-regression");
|
||||
let root = tempfile::tempdir().expect("create disk root");
|
||||
let endpoint = crate::disk::endpoint::Endpoint::try_from(root.path().to_str().expect("UTF-8 disk path"))
|
||||
.expect("parse disk endpoint");
|
||||
let local = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("open local disk"),
|
||||
);
|
||||
let wrapper = crate::disk::disk_store::LocalDiskWrapper::new(local, false);
|
||||
let disk = Arc::new(crate::disk::Disk::Local(Box::new(wrapper)));
|
||||
let error = list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![Some(disk)],
|
||||
bucket: "missing-bucket".to_owned(),
|
||||
min_disks: 1,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request.clone())
|
||||
.await
|
||||
.expect_err("missing bucket must fail the real disk producer");
|
||||
assert_eq!(error, DiskError::VolumeNotFound);
|
||||
assert!(
|
||||
producer_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"disk error must retain the request span"
|
||||
);
|
||||
|
||||
let merge_observed = Arc::new(AtomicBool::new(false));
|
||||
let merge_sink = merge_observed.clone();
|
||||
let request_id = request.id();
|
||||
list_path_raw(
|
||||
CancellationToken::new(),
|
||||
ListPathRawOptions {
|
||||
disks: vec![None],
|
||||
min_disks: 1,
|
||||
test_reader_behaviors: vec![TestReaderBehavior::Entries(vec![fallback_test_entry()])],
|
||||
agreed: Some(Box::new(move |_| {
|
||||
merge_sink.store(tracing::Span::current().id() == request_id, std::sync::atomic::Ordering::Release);
|
||||
Box::pin(async { Ok(()) })
|
||||
})),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.instrument(request)
|
||||
.await
|
||||
.expect("merge should deliver the test entry");
|
||||
assert!(
|
||||
merge_observed.load(std::sync::atomic::Ordering::Acquire),
|
||||
"merge callback must retain the request span"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_joinset_walk_uses_the_local_disk_background_gate() {
|
||||
let root = tempfile::tempdir().expect("local disk root should be created");
|
||||
@@ -1316,6 +1520,7 @@ mod tests {
|
||||
outcome.limit_reached,
|
||||
"a reader whose trailer reports a limit stop must mark the merged stream as possibly incomplete"
|
||||
);
|
||||
assert_eq!(outcome.scan_cut.as_deref(), Some("bucket/object"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -32,7 +32,6 @@ use crate::storage_api_contracts::internode::{
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_CAPABILITY_VERSION,
|
||||
};
|
||||
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use http::uri::Authority;
|
||||
use http::{HeaderMap, HeaderValue, Method, Uri};
|
||||
#[cfg(test)]
|
||||
@@ -47,6 +46,7 @@ use rustfs_io_metrics::internode_metrics::{
|
||||
global_internode_metrics,
|
||||
};
|
||||
use rustfs_object_data_cache::{MemoryBasis, resolve_effective_memory};
|
||||
use rustfs_utils::crypto::{HmacSha256, KeyInit, Mac};
|
||||
use rustfs_utils::get_env_bool;
|
||||
use sha2::Digest as _;
|
||||
use sha2::Sha256;
|
||||
@@ -58,8 +58,6 @@ use time::OffsetDateTime;
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
|
||||
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
|
||||
@@ -1590,7 +1588,13 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if boot_epoch != tonic_rpc_boot_epoch() {
|
||||
return Err(std::io::Error::other("RPC boot epoch is stale"));
|
||||
}
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)
|
||||
check_and_record_tonic_nonce(nonce, signed_at, path)?;
|
||||
// A body-bound request also carries a usable v2 signature. Consume both
|
||||
// nonces so stripping the replay-scope headers cannot authorize it again.
|
||||
if content_sha256 != UNSIGNED_PAYLOAD {
|
||||
verify_tonic_rpc_signature_with_strictness(audience, path, headers, true)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
|
||||
@@ -2581,6 +2585,43 @@ mod tests {
|
||||
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_consumes_the_body_bound_v2_nonce() {
|
||||
ensure_test_rpc_secret();
|
||||
let audience = "node-a:9000";
|
||||
let path = "/node_service.NodeService/RecoverDeleteTransaction";
|
||||
let digest = "a".repeat(64);
|
||||
for v3_first in [true, false] {
|
||||
let mut headers =
|
||||
gen_tonic_signature_headers(audience, "node_service.NodeService", "RecoverDeleteTransaction", Some(&digest))
|
||||
.expect("body-bound v2 headers should build");
|
||||
let timestamp = headers[TIMESTAMP_HEADER].to_str().expect("timestamp").to_string();
|
||||
headers.extend(
|
||||
gen_tonic_replay_scope_headers(audience, path, ×tamp, &digest, tonic_rpc_boot_epoch())
|
||||
.expect("replay-scope headers should build"),
|
||||
);
|
||||
let mut downgraded = headers.clone();
|
||||
for header in [
|
||||
RPC_REPLAY_SCOPE_VERSION_HEADER,
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
RPC_REPLAY_SCOPE_NONCE_HEADER,
|
||||
RPC_BOOT_EPOCH_HEADER,
|
||||
] {
|
||||
downgraded.remove(header);
|
||||
}
|
||||
let (first, replay) = if v3_first {
|
||||
(&headers, &downgraded)
|
||||
} else {
|
||||
(&downgraded, &headers)
|
||||
};
|
||||
verify_tonic_rpc_signature_with_policy(audience, path, first, true, false, false)
|
||||
.expect("the first delivery must be accepted");
|
||||
let error = verify_tonic_rpc_signature_with_policy(audience, path, replay, true, false, false)
|
||||
.expect_err("a mutation cannot replay through the other signature generation");
|
||||
assert_eq!(error.to_string(), "RPC request replay detected");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
|
||||
ensure_test_rpc_secret();
|
||||
|
||||
@@ -2014,9 +2014,21 @@ impl PeerRestClient {
|
||||
}
|
||||
result
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))));
|
||||
self.finalize_result(result).await
|
||||
.await;
|
||||
self.settle_bucket_metadata_reload(bucket, result).await
|
||||
}
|
||||
|
||||
async fn settle_bucket_metadata_reload(
|
||||
&self,
|
||||
bucket: &str,
|
||||
result: std::result::Result<Result<()>, tokio::time::error::Elapsed>,
|
||||
) -> Result<()> {
|
||||
match result {
|
||||
Ok(result) => self.finalize_result(result).await,
|
||||
// A local operation deadline does not establish that the peer is
|
||||
// unreachable. In particular, it must not gate IAM revocations.
|
||||
Err(_) => Err(Error::other(format!("load_bucket_metadata({bucket}) timed out"))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_bucket_metadata_once(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
|
||||
@@ -4753,6 +4765,31 @@ mod tests {
|
||||
assert!(client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bucket_metadata_deadline_does_not_gate_healthy_peer_notifications() {
|
||||
let client = test_peer_client();
|
||||
let elapsed = tokio::time::timeout(std::time::Duration::ZERO, std::future::pending::<Result<()>>()).await;
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("slow-policy", elapsed)
|
||||
.await
|
||||
.expect_err("local budget expires");
|
||||
assert!(error.to_string().contains("timed out"));
|
||||
assert!(
|
||||
!client.offline.load(Ordering::Acquire),
|
||||
"local work must not suppress subsequent IAM RPCs"
|
||||
);
|
||||
client
|
||||
.settle_bucket_metadata_reload("healthy", Ok(Ok(())))
|
||||
.await
|
||||
.expect("peer reply");
|
||||
let error = client
|
||||
.settle_bucket_metadata_reload("unreachable", Ok(Err(Error::from(tonic::Status::unavailable("connection refused")))))
|
||||
.await
|
||||
.expect_err("real transport failure");
|
||||
assert!(error.to_string().contains("connection refused"));
|
||||
assert!(client.offline.load(Ordering::Acquire), "transport failures must still enter recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn peer_rest_client_finalize_result_keeps_online_for_business_errors() {
|
||||
let client = test_peer_client();
|
||||
|
||||
@@ -67,15 +67,14 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
BatchReadVersionRequest, BatchReadVersionResponse, CheckPartsRequest, CompareAndUpdateFileOutcome,
|
||||
CompareAndUpdateFileRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest,
|
||||
DeleteVersionsResponse, DeleteVolumeRequest, DiskInfoRequest, GuardedDeleteVersionsRequest, ListDirRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, MetadataReservationActivateRequest,
|
||||
MetadataReservationCancelRequest, MetadataReservationRequest,
|
||||
PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest, PreparePartTransactionRequest, ReadAllRequest,
|
||||
ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse, ReadPartsRequest, ReadVersionRequest, ReadXlRequest,
|
||||
RecoverDeleteTransactionRequest, RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, PoolMetaNativeRecoveryRequest as PoolMetaNativeRecoveryRpcRequest,
|
||||
PreparePartTransactionRequest, ReadAllRequest, ReadMetadataRequest, ReadMultipleRequest, ReadMultipleResponse,
|
||||
ReadPartsRequest, ReadVersionRequest, ReadXlRequest, RecoverDeleteTransactionRequest,
|
||||
RecoverMultipartSourceRequest as RecoverMultipartSourceRpcRequest,
|
||||
RecoverMultipartSourceResponse as RecoverMultipartSourceRpcResponse, RenameDataRequest, RenameFileRequest,
|
||||
ReservedRenameDataRequest, SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest,
|
||||
WriteMetadataRequest, node_service_client::NodeServiceClient,
|
||||
SettlePartTransactionRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
SnapshotLeaseResponse, StatVolumeRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, WriteMetadataRequest,
|
||||
node_service_client::NodeServiceClient,
|
||||
};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use std::{
|
||||
@@ -1083,7 +1082,10 @@ impl RemoteDisk {
|
||||
if payload.len() > operation.max_wire_payload_bytes() {
|
||||
return Err(Error::other("delete recovery operation exceeds payload limit"));
|
||||
}
|
||||
let settle_timeout = if matches!(&operation, DeleteRecoveryRpcOperation::Settle { .. }) {
|
||||
let settle_timeout = if matches!(
|
||||
&operation,
|
||||
DeleteRecoveryRpcOperation::Settle { .. } | DeleteRecoveryRpcOperation::SettleBounded(_)
|
||||
) {
|
||||
Some(crate::disk::get_delete_recovery_timeout()?)
|
||||
} else {
|
||||
None
|
||||
@@ -1638,6 +1640,7 @@ impl RemoteDisk {
|
||||
let initial_probe_ok = Self::perform_connectivity_check(&addr).await.is_ok();
|
||||
if initial_probe_ok {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1709,6 +1712,7 @@ impl RemoteDisk {
|
||||
// Perform basic connectivity check
|
||||
if Self::perform_connectivity_check(&addr).await.is_ok() {
|
||||
health.record_operation_success(&endpoint, "connectivity_probe_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
} else if health.mark_failure(&endpoint, "connectivity_probe_failed") {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -1779,6 +1783,7 @@ impl RemoteDisk {
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
if Self::perform_recovery_probe(&addr, &endpoint).await.is_ok() {
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable(&addr);
|
||||
let became_online = health.mark_recovery_success(&endpoint, "disk_info_probe_success");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_HEALTH,
|
||||
@@ -2010,6 +2015,7 @@ impl RemoteDisk {
|
||||
let operation_result = operation().await;
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2024,6 +2030,7 @@ impl RemoteDisk {
|
||||
// Log success; the waiting guard balances every exit path.
|
||||
if operation_result.is_ok() {
|
||||
self.health.record_operation_success(&self.endpoint, "operation_success");
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
self.handle_network_like_error(op, timeout_duration, &operation_result, failure_health_action)
|
||||
.await;
|
||||
@@ -2453,194 +2460,6 @@ fn validate_decoded_file_info(file_info: &FileInfo) -> Result<()> {
|
||||
}
|
||||
|
||||
impl RemoteDisk {
|
||||
/// Prepare a remote token without retrying the RPC. It owns the bucket
|
||||
/// publication fence but does not enter the object metadata queue yet.
|
||||
pub async fn reserve_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"reserve_object_metadata",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
timeout_ms: u64::try_from(timeout.as_millis()).map_err(|_| Error::other("reservation timeout overflow"))?,
|
||||
bucket_incarnation_id: incarnation.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.reserve_object_metadata(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
if response.protocol_version != 2 {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
Uuid::from_slice(&response.token)
|
||||
.ok()
|
||||
.filter(|token| !token.is_nil())
|
||||
.ok_or(DiskError::Unexpected)
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token. The caller must cancel it on an indeterminate
|
||||
/// result before allowing any subsequent unprotected mutation.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid, timeout: Duration) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
self.execute_with_timeout_for_op_and_health_action(
|
||||
"activate_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationActivateRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_activate_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation activation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.activate_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
timeout.min(Duration::from_secs(5)),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed remote reservation. False cannot prove that an
|
||||
/// earlier claim did not start a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"cancel_object_metadata_reservation",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(MetadataReservationCancelRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_metadata_reservation_cancel_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reservation cancellation request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.cancel_object_metadata_reservation(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error {
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(response.success)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Publish a staged user-object write using a remote one-shot reservation.
|
||||
/// The caller must retain the bucket lifecycle fence until the tail drains.
|
||||
pub async fn rename_data_with_metadata_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
token: Uuid,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_reserved_data",
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let rename = RenameDataRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
src_volume: src_volume.to_owned(),
|
||||
src_path: src_path.to_owned(),
|
||||
file_info: compat_json(fi)?,
|
||||
dst_volume: dst_volume.to_owned(),
|
||||
dst_path: dst_path.to_owned(),
|
||||
file_info_bin: encode_file_info_msgpack(fi)?.into(),
|
||||
scanner_publication_lease_token: Bytes::new(),
|
||||
bucket_incarnation_id: Bytes::new(),
|
||||
};
|
||||
let mut request = Request::new(ReservedRenameDataRequest {
|
||||
rename: Some(rename),
|
||||
token: token.as_bytes().to_vec().into(),
|
||||
});
|
||||
let body = rustfs_protos::canonical_reserved_rename_data_request_body(request.get_ref())
|
||||
.map_err(|_| Error::other("reserved rename request length overflow"))?;
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &body).map_err(Error::other)?;
|
||||
let response = client
|
||||
.rename_reserved_data(request)
|
||||
.await
|
||||
.map_err(|status| {
|
||||
if status.code() == Code::Unimplemented {
|
||||
DiskError::UnsupportedDisk
|
||||
} else {
|
||||
status.into()
|
||||
}
|
||||
})?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.map(Into::into).unwrap_or(DiskError::Unexpected));
|
||||
}
|
||||
decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -2900,6 +2719,54 @@ impl RemoteDisk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for RemoteDisk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
use crate::disk::multipart_index::{MULTIPART_INDEX_RPC_MAX_BYTES, MultipartIndexRpcReply, MultipartIndexRpcRequest};
|
||||
let expected_disk = self
|
||||
.id
|
||||
.lock()
|
||||
.await
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or(DiskError::DiskNotFound)?;
|
||||
let query = MultipartIndexRpcRequest::new(bucket, marker);
|
||||
let endpoint = self.endpoint.to_string();
|
||||
let canonical = query.canonical_request(&endpoint)?;
|
||||
let opts = serde_json::to_string(&query)?;
|
||||
self.execute_with_timeout(
|
||||
|| async {
|
||||
let mut client = self.get_client().await?;
|
||||
let mut request = Request::new(DiskInfoRequest {
|
||||
disk: endpoint.clone(),
|
||||
opts: opts.clone(),
|
||||
});
|
||||
crate::cluster::rpc::set_tonic_canonical_body_digest(&mut request, &canonical).map_err(Error::other)?;
|
||||
let response = client.disk_info(request).await?.into_inner();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
if response.disk_info.len() > MULTIPART_INDEX_RPC_MAX_BYTES {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
// An old server ignores the additive query and returns ordinary DiskInfo.
|
||||
// It cannot satisfy the dedicated reply schema or the fresh challenge.
|
||||
let reply: MultipartIndexRpcReply =
|
||||
serde_json::from_str(&response.disk_info).map_err(|_| DiskError::MethodNotAllowed)?;
|
||||
reply.validate(&query, expected_disk)?;
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&query.canonical_response(&endpoint, &reply.page)?,
|
||||
&reply.proof,
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
Ok(reply.page)
|
||||
},
|
||||
get_drive_disk_info_timeout(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
let response = self.guarded_delete_versions_rpc("", Vec::new(), true).await?;
|
||||
if !response.errors.is_empty() || !response.item_errors.is_empty() {
|
||||
@@ -4483,6 +4350,38 @@ impl DiskAPI for RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
// Older peers reject this variant. Do not retry through the unbounded protocol;
|
||||
// they are reported as unsupported participants (rolling-upgrade contract).
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::DiscoverBounded {
|
||||
object: object.to_owned(),
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -4518,6 +4417,62 @@ impl DiskAPI for RemoteDisk {
|
||||
_ => Err(Error::other("unexpected delete discovery reply")),
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
match self
|
||||
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::SettleBounded(request))
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::SettledBounded(reply)) => Ok(reply),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
match self
|
||||
.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::InspectBounded {
|
||||
object: object.to_owned(),
|
||||
transaction,
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::InspectedBounded(page)) => Ok(page),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(error)
|
||||
if matches!(error, Error::MethodNotAllowed | Error::UnsupportedDisk)
|
||||
|| matches!(&error, Error::Io(source) if crate::cluster::rpc::client::embedded_tonic_status(source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument))) =>
|
||||
{
|
||||
// No paginated fallback through the unbounded protocol; the peer
|
||||
// is an unsupported participant until it is upgraded.
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -5038,6 +4993,113 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_lost_remote_reply_leaves_no_cursor() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("test peer must bind");
|
||||
let directory = tempfile::tempdir().expect("temporary disk");
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(
|
||||
crate::disk::local::LocalDisk::new(&endpoint, false)
|
||||
.await
|
||||
.expect("local disk"),
|
||||
);
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
let prefix = directory.path().join("bucket/prefix");
|
||||
std::fs::create_dir(&prefix).expect("prefix");
|
||||
for index in 0..64 {
|
||||
std::fs::create_dir(prefix.join(format!("child-{index}"))).expect("legitimate child");
|
||||
}
|
||||
let fixture = Arc::new(TestBoundedDiscovery {
|
||||
disk: Arc::clone(&disk),
|
||||
response_gate: TestFormatReadGate::default(),
|
||||
});
|
||||
*peer.peer.bounded_discovery.lock().expect("bounded fixture") = Some(Arc::clone(&fixture));
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let task = tokio::spawn(async move { remote.discover_delete_transactions_bounded("bucket", "prefix", 8).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), fixture.response_gate.entered.notified())
|
||||
.await
|
||||
.expect("disk completed bounded scan");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("caller cancelled").is_cancelled());
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
fixture.response_gate.release.cancel();
|
||||
peer.stop().await;
|
||||
assert_eq!(disk.public_delete_discovery_sessions_for_test(), 0);
|
||||
assert!(prefix.join("child-63").is_dir());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_old_disk_info_handler_is_not_a_ready_empty_index() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
remote
|
||||
.set_disk_id(Some(Uuid::new_v4()))
|
||||
.await
|
||||
.expect("expected disk identity");
|
||||
assert!(matches!(
|
||||
remote.list_multipart_index_roots("bucket", None).await,
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
));
|
||||
assert_eq!(
|
||||
peer.peer.disk_info_calls.load(Ordering::Acquire),
|
||||
1,
|
||||
"exercise the actual legacy JSON handler"
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bounded_delete_discovery_old_peer_rejects_without_paginated_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
assert_eq!(
|
||||
remote
|
||||
.discover_delete_transactions_bounded("bucket", "prefix", 4096)
|
||||
.await
|
||||
.expect_err("old peer cannot prove a bounded inventory"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
let transaction = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
remote
|
||||
.inspect_delete_transaction_bounded("bucket", "prefix", transaction, 65536)
|
||||
.await
|
||||
.expect_err("old peer cannot inspect with a budget"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
assert_eq!(
|
||||
remote
|
||||
.settle_delete_transaction_bounded(
|
||||
"bucket",
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "prefix".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: [0; 32],
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries: 65536,
|
||||
},
|
||||
None
|
||||
)
|
||||
.await
|
||||
.expect_err("old peer cannot settle with a budget"),
|
||||
DiskError::UnsupportedDisk
|
||||
);
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods"),
|
||||
vec!["/node_service.NodeService/RecoverDeleteTransaction"; 3]
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_recovery_old_peer_rejects_without_generic_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -5298,56 +5360,6 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_reservation_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let peer = TestGrpcPeer::spawn(Bytes::new(), Bytes::new())
|
||||
.await
|
||||
.expect("old peer must bind");
|
||||
let remote = remote_disk_for_addr(&peer.addr).await;
|
||||
let token = Uuid::new_v4();
|
||||
assert!(matches!(
|
||||
remote
|
||||
.reserve_object_metadata("bucket", "object", Uuid::new_v4(), Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.activate_object_metadata("bucket", "object", token, Duration::from_secs(5))
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote.cancel_object_metadata_reservation(token).await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert!(matches!(
|
||||
remote
|
||||
.rename_data_with_metadata_reservation(
|
||||
crate::disk::RUSTFS_META_TMP_BUCKET,
|
||||
"staged",
|
||||
&sample_rename_data_file_info(),
|
||||
"bucket",
|
||||
"object",
|
||||
token,
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
));
|
||||
assert_eq!(
|
||||
*peer.peer.unsupported_methods.lock().expect("observed methods lock"),
|
||||
vec![
|
||||
"/node_service.NodeService/ReserveObjectMetadata",
|
||||
"/node_service.NodeService/ActivateObjectMetadataReservation",
|
||||
"/node_service.NodeService/CancelObjectMetadataReservation",
|
||||
"/node_service.NodeService/RenameReservedData",
|
||||
],
|
||||
"reservation operations must never downgrade to legacy methods",
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_fence_old_peer_rejects_without_legacy_fallback() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
@@ -5979,10 +5991,18 @@ mod tests {
|
||||
object_read_all_disks: Arc<StdMutex<Vec<String>>>,
|
||||
format_data: Bytes,
|
||||
read_all_data: Bytes,
|
||||
fail_format_reads: Arc<AtomicBool>,
|
||||
next_format_read_gate: Arc<StdMutex<Option<Arc<TestFormatReadGate>>>>,
|
||||
bounded_discovery: Arc<StdMutex<Option<Arc<TestBoundedDiscovery>>>>,
|
||||
unsupported_methods: Arc<StdMutex<Vec<String>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct TestBoundedDiscovery {
|
||||
disk: Arc<crate::disk::local::LocalDisk>,
|
||||
response_gate: TestFormatReadGate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct TestFormatReadGate {
|
||||
entered: tokio::sync::Notify,
|
||||
@@ -5998,7 +6018,9 @@ mod tests {
|
||||
object_read_all_disks: Arc::default(),
|
||||
format_data,
|
||||
read_all_data,
|
||||
fail_format_reads: Arc::default(),
|
||||
next_format_read_gate: Arc::default(),
|
||||
bounded_discovery: Arc::default(),
|
||||
unsupported_methods: Arc::default(),
|
||||
}
|
||||
}
|
||||
@@ -6098,6 +6120,56 @@ mod tests {
|
||||
Ok(grpc.unary(method, request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/RecoverDeleteTransaction"
|
||||
if self.peer.bounded_discovery.lock().expect("bounded fixture").is_some() =>
|
||||
{
|
||||
#[derive(Clone)]
|
||||
struct BoundedSvc(Arc<TestBoundedDiscovery>);
|
||||
impl tonic::server::UnaryService<RecoverDeleteTransactionRequest> for BoundedSvc {
|
||||
type Response = rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse;
|
||||
type Future = Pin<Box<dyn Future<Output = std::result::Result<Response<Self::Response>, Status>> + Send>>;
|
||||
fn call(&mut self, request: Request<RecoverDeleteTransactionRequest>) -> Self::Future {
|
||||
let fixture = Arc::clone(&self.0);
|
||||
Box::pin(async move {
|
||||
let request = request.into_inner();
|
||||
let operation: DeleteRecoveryRpcOperation =
|
||||
rustfs_protos::decode_delete_recovery_payload(&request.operation)
|
||||
.map_err(|_| Status::invalid_argument("invalid bounded request"))?;
|
||||
let DeleteRecoveryRpcOperation::DiscoverBounded { object, max_entries } = operation else {
|
||||
return Err(Status::invalid_argument("unexpected fallback"));
|
||||
};
|
||||
let page = fixture
|
||||
.disk
|
||||
.discover_delete_transactions_bounded(&request.volume, &object, max_entries)
|
||||
.await
|
||||
.map_err(|_| Status::internal("bounded discovery failed"))?;
|
||||
// Lose the response only after the real disk operation has completed.
|
||||
fixture.response_gate.entered.notify_one();
|
||||
fixture.response_gate.release.cancelled().await;
|
||||
let result = rmp_serde::to_vec_named(&DeleteRecoveryRpcReply::DiscoveredBounded(page))
|
||||
.map_err(|_| Status::internal("encode bounded reply"))?;
|
||||
Ok(Response::new(rustfs_protos::proto_gen::node_service::RecoverDeleteTransactionResponse {
|
||||
success: true,
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
result: result.into(),
|
||||
error: None,
|
||||
}))
|
||||
})
|
||||
}
|
||||
}
|
||||
let fixture = self
|
||||
.peer
|
||||
.bounded_discovery
|
||||
.lock()
|
||||
.expect("bounded fixture")
|
||||
.clone()
|
||||
.expect("configured fixture");
|
||||
Box::pin(async move {
|
||||
let codec = tonic_prost::ProstCodec::default();
|
||||
let mut grpc = tonic::server::Grpc::new(codec);
|
||||
Ok(grpc.unary(BoundedSvc(fixture), request).await)
|
||||
})
|
||||
}
|
||||
"/node_service.NodeService/ReadAll" => {
|
||||
#[derive(Clone)]
|
||||
struct ReadAllSvc(AuthenticatedReadPeer);
|
||||
@@ -6121,6 +6193,9 @@ mod tests {
|
||||
gate.entered.notify_one();
|
||||
gate.release.cancelled().await;
|
||||
}
|
||||
if peer.fail_format_reads.load(Ordering::Acquire) {
|
||||
return Err(Status::unavailable("format read temporarily unavailable"));
|
||||
}
|
||||
peer.format_data.clone()
|
||||
} else {
|
||||
peer.object_read_all_disks
|
||||
@@ -9086,6 +9161,124 @@ mod tests {
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_failure_preserves_registered_remote_recovery() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(crate::disk::disk_store::ENV_RUSTFS_DRIVE_ACTIVE_MONITORING, Some("true")),
|
||||
(rustfs_config::ENV_DRIVE_RETURNING_SUCCESS_THRESHOLD, Some("1")),
|
||||
],
|
||||
async {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(
|
||||
Bytes::from(reference.to_json().expect("format should serialize")),
|
||||
Bytes::from_static(b"recovered-peer-data"),
|
||||
)
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
let disk = crate::disk::new_disk(
|
||||
&set.set_endpoints[0],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("registered remote disk should construct");
|
||||
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
|
||||
panic!("fixture must create a remote disk");
|
||||
};
|
||||
assert!(remote.health_check);
|
||||
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
set.disks.write().await[0] = Some(Arc::clone(&disk));
|
||||
|
||||
peer.peer.fail_format_reads.store(true, Ordering::Release);
|
||||
set.connect_disks().await;
|
||||
peer.peer.fail_format_reads.store(false, Ordering::Release);
|
||||
let cancelled_after_failure = remote.cancel_token.is_cancelled();
|
||||
assert!(peer.peer.read_all_calls() > 0, "reconnect must attempt the failing format RPC");
|
||||
assert!(
|
||||
Arc::ptr_eq(&disk, set.disks.read().await[0].as_ref().expect("existing slot must remain")),
|
||||
"a failed reconnect must not publish its unvalidated candidate"
|
||||
);
|
||||
remote.spawn_recovery_monitor_if_needed();
|
||||
let recovery = time::timeout(Duration::from_secs(5), async {
|
||||
while !disk.is_online().await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
disk.read_all("bucket", "object").await
|
||||
})
|
||||
.await;
|
||||
disk.close().await.expect("stop the registered disk's monitoring");
|
||||
peer.stop().await;
|
||||
assert!(!cancelled_after_failure, "failed replacement must retain the registered monitor");
|
||||
assert_eq!(
|
||||
recovery
|
||||
.expect("failed reconnect must leave the registered disk able to recover without another sweep")
|
||||
.expect("the recovered registered disk must serve real I/O"),
|
||||
Bytes::from_static(b"recovered-peer-data")
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_stops_old_remote_monitor_only_after_publication() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let mut reference = crate::layout::format::FormatV3::new(1, 1);
|
||||
reference.erasure.this = reference.erasure.sets[0][0];
|
||||
let peer = TestGrpcPeer::spawn(
|
||||
Bytes::from(reference.to_json().expect("format should serialize")),
|
||||
Bytes::from_static(b"replacement-peer-data"),
|
||||
)
|
||||
.await
|
||||
.expect("authenticated regression peer is required");
|
||||
let set = missing_read_set(&peer, reference).await;
|
||||
let disk = crate::disk::new_disk(
|
||||
&set.set_endpoints[0],
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("registered remote disk should construct");
|
||||
let crate::disk::Disk::Remote(remote) = disk.as_ref() else {
|
||||
panic!("fixture must create a remote disk");
|
||||
};
|
||||
remote.force_runtime_state_for_test(RuntimeDriveHealthState::Offline);
|
||||
set.disks.write().await[0] = Some(Arc::clone(&disk));
|
||||
let gate = peer.peer.pause_next_format_read();
|
||||
let pending_set = Arc::clone(&set);
|
||||
let pending = tokio::spawn(async move { pending_set.connect_disks().await });
|
||||
time::timeout(Duration::from_secs(5), gate.entered.notified())
|
||||
.await
|
||||
.expect("replacement must reach the format validation barrier");
|
||||
let cancelled_before_publication = remote.cancel_token.is_cancelled();
|
||||
gate.release.cancel();
|
||||
pending.await.expect("validated replacement must finish publication");
|
||||
let replacement = set.disks.read().await[0].clone().expect("replacement must occupy the slot");
|
||||
let readback = replacement.read_all("bucket", "object").await;
|
||||
replacement.close().await.expect("stop replacement monitoring");
|
||||
peer.stop().await;
|
||||
assert!(
|
||||
!cancelled_before_publication,
|
||||
"the registered monitor must survive pending format validation"
|
||||
);
|
||||
assert!(!Arc::ptr_eq(&disk, &replacement), "validated replacement must be published");
|
||||
assert!(remote.cancel_token.is_cancelled(), "the displaced monitor must stop after publication");
|
||||
assert_eq!(
|
||||
readback.expect("replacement must serve real I/O"),
|
||||
Bytes::from_static(b"replacement-peer-data")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(remote_disk_recovery_probe)]
|
||||
async fn endpoint_reconnect_admits_ready_drive_while_another_format_probe_is_blocked() {
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::cluster::rpc::client::{
|
||||
use crate::cluster::rpc::set_tonic_rolling_mutation_body_digest;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use futures::{FutureExt, future::BoxFuture};
|
||||
use rustfs_lock::{
|
||||
LockClient, LockError, LockInfo, LockRequest, LockResponse, LockStats, LockStatus, LockType, Result,
|
||||
types::{LockId, LockMetadata, LockPriority},
|
||||
@@ -32,16 +33,17 @@ use rustfs_protos::{
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::panic::AssertUnwindSafe;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::{Instant, timeout};
|
||||
use tonic::service::interceptor::InterceptedService;
|
||||
use tonic::{Request, Response};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const EVENT_NAMESPACE_LOCK_RELEASE: &str = "namespace_lock_release";
|
||||
const EVENT_REMOTE_LOCK_RPC_CLEANUP: &str = "remote_lock_rpc_cleanup";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_REMOTE_LOCK: &str = "remote_lock";
|
||||
|
||||
@@ -49,18 +51,38 @@ fn attach_lock_mutation_body_digest<T: rustfs_protos::CanonicalMutationBody>(req
|
||||
set_tonic_rolling_mutation_body_digest(request)
|
||||
}
|
||||
|
||||
/// Work to run if an RPC that already timed out for its caller completes later.
|
||||
/// Work to run if an RPC completes after its caller stops waiting.
|
||||
type LateCompletion<T> = Option<Box<dyn FnOnce(T) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send>>;
|
||||
type LateFailure = LateCompletion<()>;
|
||||
type RpcFuture<T> = BoxFuture<'static, std::thread::Result<std::result::Result<T, tonic::Status>>>;
|
||||
|
||||
struct RpcCleanup<T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
/// Own admission before connecting as well as while an RPC is in flight.
|
||||
struct LockRequestGuard<'a> {
|
||||
client: &'a RemoteClient,
|
||||
op: &'static str,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
}
|
||||
|
||||
impl<T> RpcCleanup<T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestAdmission>) -> Self {
|
||||
impl LockRequestGuard<'_> {
|
||||
fn into_admission(mut self) -> Option<LockRequestAdmission> {
|
||||
self.admission.take()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for LockRequestGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.client.release_lock_request(self.op, self.admission.take());
|
||||
}
|
||||
}
|
||||
|
||||
struct RpcCleanup<'a, T> {
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestGuard<'a>>,
|
||||
}
|
||||
|
||||
impl<'a, T> RpcCleanup<'a, T> {
|
||||
fn new(late: LateCompletion<T>, late_failure: LateFailure, admission: Option<LockRequestGuard<'a>>) -> Self {
|
||||
Self {
|
||||
late,
|
||||
late_failure,
|
||||
@@ -69,6 +91,25 @@ impl<T> RpcCleanup<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A dropped caller hands its RPC and cleanup to the same bounded owner as a timeout.
|
||||
struct InFlightRpc<'a, T: Send + 'static> {
|
||||
client: &'a RemoteClient,
|
||||
op: &'static str,
|
||||
resource_summary: &'a str,
|
||||
future: Option<RpcFuture<T>>,
|
||||
cleanup: Option<RpcCleanup<'a, T>>,
|
||||
}
|
||||
|
||||
impl<T: Send + 'static> Drop for InFlightRpc<'_, T> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(future) = self.future.take()
|
||||
&& let Some(cleanup) = self.cleanup.take()
|
||||
{
|
||||
self.client.detach_rpc(self.op, self.resource_summary, future, cleanup);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The liveness window is this many RPC deadlines: a peer that completed a
|
||||
/// lock RPC within it is slow, not gone, and keeps its channel on a timeout.
|
||||
const LOCK_RPC_LIVENESS_WINDOW_DEADLINES: u32 = 2;
|
||||
@@ -137,6 +178,9 @@ fn with_lock_peer_health<R>(addr: &str, update: impl FnOnce(&mut LockPeerChannel
|
||||
let mut peers = lock_peer_channel_health()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(health) = peers.get_mut(addr) {
|
||||
return update(health);
|
||||
}
|
||||
update(peers.entry(addr.to_string()).or_default())
|
||||
}
|
||||
|
||||
@@ -158,6 +202,15 @@ fn reset_lock_peer_health_for_test(addr: &str) {
|
||||
.remove(addr);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type EvictionPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
#[cfg(test)]
|
||||
fn eviction_pauses_for_test() -> &'static Mutex<HashMap<String, EvictionPause>> {
|
||||
static PAUSES: OnceLock<Mutex<HashMap<String, EvictionPause>>> = OnceLock::new();
|
||||
PAUSES.get_or_init(Mutex::default)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum EvictionTrigger {
|
||||
/// The caller's deadline expired while the stream was still open.
|
||||
@@ -445,6 +498,10 @@ impl RemoteClient {
|
||||
}
|
||||
}
|
||||
});
|
||||
// A successful lock RPC over a cached or fresh channel proves the peer is
|
||||
// reachable; reconcile the internode peer-health gauge so it does not stay
|
||||
// stale when no fresh dial fires (issue #9119).
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&self.addr);
|
||||
}
|
||||
|
||||
fn admit_late_release(&self) -> bool {
|
||||
@@ -502,6 +559,14 @@ impl RemoteClient {
|
||||
admission
|
||||
}
|
||||
|
||||
fn admit_lock_request_guard(&self, op: &'static str) -> Option<LockRequestGuard<'_>> {
|
||||
self.admit_lock_request(op).map(|admission| LockRequestGuard {
|
||||
client: self,
|
||||
op,
|
||||
admission: Some(admission),
|
||||
})
|
||||
}
|
||||
|
||||
fn release_lock_request(&self, op: &'static str, admission: Option<LockRequestAdmission>) {
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
with_lock_peer_health(&self.addr, |health| {
|
||||
@@ -583,6 +648,17 @@ impl RemoteClient {
|
||||
trigger: EvictionTrigger,
|
||||
deadline: Duration,
|
||||
) {
|
||||
#[cfg(test)]
|
||||
let pause = eviction_pauses_for_test()
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&self.addr);
|
||||
#[cfg(test)]
|
||||
if let Some((entered, resume)) = pause {
|
||||
let _ = entered.send(());
|
||||
let _ = resume.await;
|
||||
}
|
||||
|
||||
let now = Instant::now();
|
||||
let cooldown = Self::eviction_cooldown();
|
||||
let liveness_window = Self::liveness_window(deadline);
|
||||
@@ -613,22 +689,26 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
|
||||
/// Keep a timed-out RPC running instead of cancelling its stream.
|
||||
/// Keep an abandoned RPC running instead of cancelling its stream.
|
||||
///
|
||||
/// Dropping the future sends `RST_STREAM`; under load those resets pile up
|
||||
/// in the server's pending-accept queue until it answers `GOAWAY
|
||||
/// too_many_resets` and kills every stream on the connection. A detached
|
||||
/// stream ends on its own within the internode RPC timeout, the number per
|
||||
/// peer is bounded, and a lock granted after its caller gave up is released.
|
||||
fn detach_timed_out_rpc<T: Send + 'static>(
|
||||
fn detach_rpc<T: Send + 'static>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
resource_summary: &str,
|
||||
handle: JoinHandle<std::result::Result<T, tonic::Status>>,
|
||||
late: LateCompletion<T>,
|
||||
late_failure: LateFailure,
|
||||
admission: Option<LockRequestAdmission>,
|
||||
future: RpcFuture<T>,
|
||||
cleanup: RpcCleanup<'_, T>,
|
||||
) {
|
||||
let RpcCleanup {
|
||||
late,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let admission = admission.and_then(LockRequestGuard::into_admission);
|
||||
let limit = Self::detached_rpc_limit();
|
||||
let admitted = with_lock_peer_health(&self.addr, |health| {
|
||||
if health.detached_rpcs >= limit {
|
||||
@@ -639,25 +719,29 @@ impl RemoteClient {
|
||||
}
|
||||
});
|
||||
if !admitted {
|
||||
handle.abort();
|
||||
drop(future);
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "aborted");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "detached_budget_exhausted",
|
||||
addr = %self.addr,
|
||||
op,
|
||||
resource_summary,
|
||||
limit,
|
||||
"Cancelled timed-out remote lock RPC because the detached stream budget is exhausted"
|
||||
"Abandoned remote lock RPC cancelled"
|
||||
);
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_detached(op, "detached");
|
||||
let addr = self.addr.clone();
|
||||
tokio::spawn(async move {
|
||||
let outcome = handle.await;
|
||||
let outcome = future.await;
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.detached_rpcs = health.detached_rpcs.saturating_sub(1);
|
||||
if is_request_breaker_operation(op) && admission.is_some() {
|
||||
@@ -672,6 +756,7 @@ impl RemoteClient {
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.last_success = Some(Instant::now());
|
||||
});
|
||||
rustfs_io_metrics::internode_metrics::record_peer_reachable_if_offline(&addr);
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "success");
|
||||
if let Some(late) = late {
|
||||
late(response).await;
|
||||
@@ -683,19 +768,31 @@ impl RemoteClient {
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "error");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_rpc_failed",
|
||||
addr = %addr,
|
||||
op,
|
||||
tonic_code = ?status.code(),
|
||||
tonic_message = status.message(),
|
||||
"Detached remote lock RPC failed after its caller timed out"
|
||||
"Abandoned remote lock RPC failed"
|
||||
);
|
||||
}
|
||||
Err(join_error) => {
|
||||
Err(_) => {
|
||||
if let Some(late_failure) = late_failure {
|
||||
late_failure(()).await;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_late_completion(op, "join_error");
|
||||
debug!(addr = %addr, op, error = %join_error, "Detached remote lock RPC task ended abnormally");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_rpc_panicked",
|
||||
addr = %addr,
|
||||
op,
|
||||
"Abandoned remote lock RPC panicked"
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -783,16 +880,28 @@ impl RemoteClient {
|
||||
self.release_late_release();
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_late_release(outcome);
|
||||
if outcome == "released" {
|
||||
debug!(addr = %self.addr, count = original_count, "Released remote locks granted after their caller timed out");
|
||||
debug!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_grants_released",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
"Released remote locks granted after their caller stopped waiting"
|
||||
);
|
||||
} else {
|
||||
let (log_warning, suppressed_timeouts) = self.timeout_log_decision("late_release", Self::rpc_timeout());
|
||||
if log_warning {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "late_grants_unreleased",
|
||||
addr = %self.addr,
|
||||
count = original_count,
|
||||
outcome,
|
||||
suppressed_timeouts,
|
||||
"Could not release every remote lock granted after its caller timed out; the server lease will expire it"
|
||||
"Could not release every remote lock granted after its caller stopped waiting; the server lease will expire it"
|
||||
);
|
||||
} else {
|
||||
debug!(addr = %self.addr, count = original_count, outcome, suppressed_timeouts, "Suppressed repeated late remote lock release warning");
|
||||
@@ -806,25 +915,47 @@ impl RemoteClient {
|
||||
resource_summary: &str,
|
||||
deadline: Duration,
|
||||
future: Fut,
|
||||
cleanup: RpcCleanup<T>,
|
||||
cleanup: RpcCleanup<'_, T>,
|
||||
) -> std::result::Result<T, LockError>
|
||||
where
|
||||
Fut: Future<Output = std::result::Result<T, tonic::Status>> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let RpcCleanup {
|
||||
late,
|
||||
let mut rpc = InFlightRpc {
|
||||
client: self,
|
||||
op,
|
||||
resource_summary,
|
||||
future: Some(AssertUnwindSafe(future).catch_unwind().boxed()),
|
||||
cleanup: Some(cleanup),
|
||||
};
|
||||
let outcome = match rpc.future.as_mut() {
|
||||
Some(future) => tokio::select! {
|
||||
// A response becoming ready on the deadline tick must not turn an
|
||||
// expired caller budget into a transport error and channel eviction.
|
||||
biased;
|
||||
_ = tokio::time::sleep(deadline) => None,
|
||||
outcome = future => Some(outcome),
|
||||
},
|
||||
None => return Err(LockError::internal("remote lock RPC has no in-flight future")),
|
||||
};
|
||||
if outcome.is_some() {
|
||||
rpc.future.take();
|
||||
}
|
||||
let Some(RpcCleanup {
|
||||
late: _,
|
||||
late_failure,
|
||||
admission,
|
||||
} = cleanup;
|
||||
let mut handle = tokio::spawn(future);
|
||||
match timeout(deadline, &mut handle).await {
|
||||
Ok(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, admission);
|
||||
self.release_lock_request(op, admission);
|
||||
}) = rpc.cleanup.as_mut()
|
||||
else {
|
||||
return Err(LockError::internal("remote lock RPC has no cleanup owner"));
|
||||
};
|
||||
let token = admission.as_ref().and_then(|guard| guard.admission);
|
||||
let result = match outcome {
|
||||
Some(Ok(Ok(response))) => {
|
||||
self.record_rpc_success(op, token);
|
||||
Ok(response)
|
||||
}
|
||||
Ok(Ok(Err(err))) => {
|
||||
Some(Ok(Err(err))) => {
|
||||
let reason = err.to_string();
|
||||
if err.code() == tonic::Code::DeadlineExceeded {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
@@ -857,12 +988,11 @@ impl RemoteClient {
|
||||
"Suppressed repeated remote lock RPC deadline"
|
||||
);
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
// Only evict (and re-dial) the cached channel when the failure is a genuine
|
||||
@@ -923,30 +1053,32 @@ impl RemoteClient {
|
||||
);
|
||||
}
|
||||
if transport_failure {
|
||||
// Ambiguous-grant cleanup owns its work before channel eviction can suspend.
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Transport, deadline)
|
||||
.await;
|
||||
}
|
||||
self.release_lock_request(op, admission);
|
||||
if transport_failure && let Some(late_failure) = late_failure {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC failed: {reason}")))
|
||||
}
|
||||
Ok(Err(join_error)) => {
|
||||
Some(Err(_)) => {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_LOCK_RPC_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_LOCK,
|
||||
state = "rpc_panicked",
|
||||
addr = %self.addr,
|
||||
op,
|
||||
resource_summary,
|
||||
error = %join_error,
|
||||
"Remote lock RPC task ended abnormally"
|
||||
"Remote lock RPC panicked"
|
||||
);
|
||||
self.release_lock_request(op, admission);
|
||||
if let Some(late_failure) = late_failure {
|
||||
if let Some(late_failure) = late_failure.take() {
|
||||
drop(tokio::spawn(late_failure(())));
|
||||
}
|
||||
Err(LockError::internal(format!("{op} RPC task failed: {join_error}")))
|
||||
Err(LockError::internal(format!("{op} RPC future panicked")))
|
||||
}
|
||||
Err(_) => {
|
||||
None => {
|
||||
let reason = format!("RPC timed out after {deadline:?}");
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_rpc_timeout(&self.addr, op);
|
||||
self.record_request_timeout(op, deadline);
|
||||
@@ -978,12 +1110,17 @@ impl RemoteClient {
|
||||
"Suppressed repeated remote lock RPC timeout"
|
||||
);
|
||||
}
|
||||
// Continue draining the abandoned stream while channel eviction waits.
|
||||
drop(rpc);
|
||||
self.maybe_evict_connection(op, &reason, resource_summary, EvictionTrigger::Timeout, deadline)
|
||||
.await;
|
||||
self.detach_timed_out_rpc(op, resource_summary, handle, late, late_failure, admission);
|
||||
Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline))
|
||||
return Err(LockError::timeout(format!("remote lock RPC {op} on {}", self.addr), deadline));
|
||||
}
|
||||
};
|
||||
if rpc.future.is_none() {
|
||||
rpc.cleanup.take();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn rpc_timeout_failure_response(request: &LockRequest, err: &LockError) -> LockResponse {
|
||||
@@ -1074,7 +1211,7 @@ impl RemoteClient {
|
||||
#[async_trait]
|
||||
impl LockClient for RemoteClient {
|
||||
async fn acquire_lock(&self, request: &LockRequest) -> Result<LockResponse> {
|
||||
let admission = match self.admit_lock_request("lock") {
|
||||
let admission = match self.admit_lock_request_guard("lock") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_response(request)),
|
||||
};
|
||||
@@ -1085,7 +1222,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock", rpc_timeout);
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
@@ -1093,14 +1229,12 @@ impl LockClient for RemoteClient {
|
||||
let args = match serde_json::to_string(&bounded_request) {
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(LockError::internal(format!("Failed to serialize request: {err}")));
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(GenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_hook(request.lock_id.clone());
|
||||
@@ -1142,7 +1276,7 @@ impl LockClient for RemoteClient {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let admission = match self.admit_lock_request("lock_batch") {
|
||||
let admission = match self.admit_lock_request_guard("lock_batch") {
|
||||
Some(admission) => admission,
|
||||
None => return Ok(Self::rpc_backoff_failure_batch(requests)),
|
||||
};
|
||||
@@ -1156,7 +1290,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
self.record_request_timeout("lock_batch", rpc_timeout);
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
@@ -1170,14 +1303,12 @@ impl LockClient for RemoteClient {
|
||||
{
|
||||
Ok(args) => args,
|
||||
Err(err) => {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let mut req = Request::new(BatchGenerallyLockRequest { args });
|
||||
req.set_timeout(rpc_timeout);
|
||||
if let Err(err) = attach_lock_mutation_body_digest(&mut req) {
|
||||
self.release_lock_request("lock_batch", Some(admission));
|
||||
return Err(err.into());
|
||||
}
|
||||
let late = self.late_release_batch_hook(requests.iter().map(|request| request.lock_id.clone()).collect());
|
||||
@@ -1315,122 +1446,6 @@ impl LockClient for RemoteClient {
|
||||
Ok(resp.success)
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_CONVERSION_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
// Older nodes refresh normally but return no conversion marker.
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn try_upgrade_committed_tail(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::COMMITTED_TAIL_UPGRADE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn supports_heal_lock_transition(&self, lock_id: &LockId) -> Result<bool> {
|
||||
let marker = rustfs_lock::client::HEAL_LOCK_TRANSITION_PROBE_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
Ok(response.success && response.lock_info.as_deref() == Some(marker))
|
||||
}
|
||||
|
||||
async fn waiting_writer(&self, lock_id: &LockId) -> Result<Option<bool>> {
|
||||
let marker = rustfs_lock::client::HEAL_WAITING_WRITER_MARKER;
|
||||
let mut request = Self::create_unlock_request(lock_id);
|
||||
request.metadata.tags.insert(marker.to_owned(), marker.to_owned());
|
||||
let mut client = self.get_client().await?;
|
||||
let resource_summary = request.resource.to_string();
|
||||
let mut req = Request::new(GenerallyLockRequest {
|
||||
args: serde_json::to_string(&request)
|
||||
.map_err(|error| LockError::internal(format!("Failed to serialize request: {error}")))?,
|
||||
});
|
||||
attach_lock_mutation_body_digest(&mut req)?;
|
||||
let response = self
|
||||
.execute_rpc(
|
||||
"refresh",
|
||||
&resource_summary,
|
||||
Self::rpc_timeout(),
|
||||
async move { client.refresh(req).await },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await?
|
||||
.into_inner();
|
||||
if let Some(error) = response.error_info {
|
||||
return Err(LockError::internal(error));
|
||||
}
|
||||
if !response.success {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(match response.lock_info.as_deref().and_then(|info| info.strip_prefix(marker)) {
|
||||
Some(":0") => Some(false),
|
||||
Some(":1") => Some(true),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &LockId) -> Result<bool> {
|
||||
info!("remote force_release for {}", lock_id);
|
||||
let force_request = Self::create_unlock_request(lock_id);
|
||||
@@ -1821,6 +1836,351 @@ mod tests {
|
||||
assert!(acquired_lock_ids(&lock_ids, &[]).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_connection_releases_request_admission() {
|
||||
ensure_test_rpc_secret();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind pending-handshake peer");
|
||||
let addr = format!("http://{}", listener.local_addr().expect("pending-handshake peer address"));
|
||||
let client = RemoteClient::new(addr.clone());
|
||||
let request = test_lock_request(Duration::from_secs(30));
|
||||
|
||||
for batch in [false, true] {
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
with_lock_peer_health(&addr, |health| {
|
||||
health.request_backoff_until = Some(Instant::now() - Duration::from_secs(1));
|
||||
});
|
||||
let mut acquisition = Box::pin(async {
|
||||
if batch {
|
||||
client.acquire_locks_batch(std::slice::from_ref(&request)).await.map(|_| ())
|
||||
} else {
|
||||
client.acquire_lock(&request).await.map(|_| ())
|
||||
}
|
||||
});
|
||||
std::future::poll_fn(|cx| {
|
||||
assert!(acquisition.as_mut().poll(cx).is_pending(), "peer must wait for its HTTP/2 handshake");
|
||||
std::task::Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
let pending = lock_peer_health_for_test(&addr);
|
||||
assert_eq!(pending.request_in_flight, 1, "pending connection owns one request admission");
|
||||
assert!(pending.request_probe_generation.is_some(), "the request is the half-open probe");
|
||||
|
||||
drop(acquisition);
|
||||
let cancelled = lock_peer_health_for_test(&addr);
|
||||
assert_eq!(cancelled.request_in_flight, 0, "cancellation must return request admission");
|
||||
assert!(cancelled.request_probe_generation.is_none(), "cancellation must unblock the next probe");
|
||||
let probe = client.admit_lock_request("lock").expect("cancelled peer can be probed again");
|
||||
client.release_lock_request("lock", Some(probe));
|
||||
}
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_rpc_runs_late_grant_cleanup() {
|
||||
let addr = "http://cancelled-rpc-late-grant";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(granted).expect("late grant observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move {
|
||||
started_tx.send(()).expect("caller observes RPC dispatch");
|
||||
Ok(grant_rx.await.expect("test decides when peer grants lock"))
|
||||
},
|
||||
RpcCleanup::new(late, None, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.expect("RPC must dispatch")
|
||||
.expect("dispatch signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
1,
|
||||
"cancelled RPC uses bounded detached ownership"
|
||||
);
|
||||
grant_tx.send(true).expect("cancellation must preserve the running RPC");
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("late grant must be cleaned up")
|
||||
.expect("cleanup result")
|
||||
);
|
||||
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "completed cleanup returns its slot");
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_rpc_runs_ambiguous_failure_cleanup() {
|
||||
let addr = "http://cancelled-rpc-late-failure";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (failure_tx, failure_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("ambiguous cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock_batch",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move {
|
||||
started_tx.send(()).expect("caller observes RPC dispatch");
|
||||
failure_rx.await.expect("test decides when transport fails");
|
||||
Err::<(), _>(tonic::Status::cancelled("transport failure after caller cancellation"))
|
||||
},
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), started_rx)
|
||||
.await
|
||||
.expect("RPC must dispatch")
|
||||
.expect("dispatch signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller was cancelled").is_cancelled());
|
||||
failure_tx.send(()).expect("cancellation must preserve the running RPC");
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("ambiguous grant must be cleaned up")
|
||||
.expect("cleanup signal");
|
||||
assert_eq!(lock_peer_health_for_test(addr).detached_rpcs, 0, "failed detached RPC returns its slot");
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_cancelled_completed_failure_runs_cleanup() {
|
||||
for (label, status) in [
|
||||
("transport", tonic::Status::cancelled("ambiguous transport failure")),
|
||||
("deadline", tonic::Status::deadline_exceeded("ambiguous deadline failure")),
|
||||
] {
|
||||
let addr = format!("http://cancelled-completed-{label}");
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
let client = RemoteClient::new(addr.clone());
|
||||
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
|
||||
let (_resume_tx, resume_rx) = tokio::sync::oneshot::channel();
|
||||
eviction_pauses_for_test()
|
||||
.lock()
|
||||
.expect("eviction pause registry")
|
||||
.insert(addr.clone(), (evicting_tx, resume_rx));
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx
|
||||
.send(())
|
||||
.expect("completed-failure cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(30),
|
||||
async move { Err::<(), _>(status) },
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), evicting_rx)
|
||||
.await
|
||||
.expect("completed error must reach channel eviction")
|
||||
.expect("eviction signal");
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("cleanup must transfer before channel eviction suspends")
|
||||
.expect("cleanup signal");
|
||||
caller.abort();
|
||||
assert!(caller.await.expect_err("caller is cancelled during eviction").is_cancelled());
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(&addr).detached_rpcs,
|
||||
0,
|
||||
"completed RPC cannot consume a detached slot"
|
||||
);
|
||||
reset_lock_peer_health_for_test(&addr);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_remote_client_deadline_wins_over_same_tick_transport_timeout() {
|
||||
let addr = "http://same-tick-rpc-deadline";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
with_lock_peer_health(addr, |health| health.last_success = Some(Instant::now()));
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let budget = Duration::from_millis(50);
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("same-tick cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
let result = client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
budget,
|
||||
async move {
|
||||
tokio::time::sleep(budget).await;
|
||||
Err::<(), _>(tonic::Status::cancelled("inner transport deadline expired"))
|
||||
},
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Timeout { .. })),
|
||||
"expired caller budget must retain timeout classification"
|
||||
);
|
||||
let health = lock_peer_health_for_test(addr);
|
||||
assert_eq!(health.consecutive_timeouts, 1, "same-tick completion must follow timeout eviction policy");
|
||||
assert!(
|
||||
health.last_eviction.is_none(),
|
||||
"a recently serving peer keeps its channel after a caller timeout"
|
||||
);
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("same-tick ambiguous completion must trigger cleanup")
|
||||
.expect("cleanup signal");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
0,
|
||||
"same-tick completion returns detached admission"
|
||||
);
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_timeout_drains_late_grant_during_eviction() {
|
||||
let addr = "http://timeout-late-grant-during-eviction";
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
let client = RemoteClient::new(addr.to_string());
|
||||
let (evicting_tx, evicting_rx) = tokio::sync::oneshot::channel();
|
||||
let (resume_tx, resume_rx) = tokio::sync::oneshot::channel();
|
||||
eviction_pauses_for_test()
|
||||
.lock()
|
||||
.expect("eviction pause registry")
|
||||
.insert(addr.to_string(), (evicting_tx, resume_rx));
|
||||
let (grant_tx, grant_rx) = tokio::sync::oneshot::channel();
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let late: LateCompletion<bool> = Some(Box::new(move |granted| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(granted).expect("late grant observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_millis(25),
|
||||
async move { Ok(grant_rx.await.expect("test controls the late grant")) },
|
||||
RpcCleanup::new(late, None, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
timeout(Duration::from_secs(1), evicting_rx)
|
||||
.await
|
||||
.expect("timed-out RPC must reach channel eviction")
|
||||
.expect("eviction signal");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
1,
|
||||
"eviction waits with a detached RPC owner"
|
||||
);
|
||||
grant_tx
|
||||
.send(true)
|
||||
.expect("timed-out stream remains available for a late grant");
|
||||
assert!(
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("late grant must be cleaned up before eviction resumes")
|
||||
.expect("late cleanup result")
|
||||
);
|
||||
assert!(!caller.is_finished(), "channel eviction is still paused");
|
||||
assert_eq!(
|
||||
lock_peer_health_for_test(addr).detached_rpcs,
|
||||
0,
|
||||
"late completion returns detached admission"
|
||||
);
|
||||
resume_tx.send(()).expect("resume channel eviction");
|
||||
let result = caller.await.expect("timed-out caller completes without panicking");
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Timeout { .. })),
|
||||
"caller preserves its RPC timeout result"
|
||||
);
|
||||
reset_lock_peer_health_for_test(addr);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_success_polls_rpc_on_callers_task() {
|
||||
tokio::spawn(async {
|
||||
let client = RemoteClient::new("http://direct-poll-success".to_string());
|
||||
let caller_id = tokio::task::id();
|
||||
let rpc_id = client
|
||||
.execute_rpc(
|
||||
"ping",
|
||||
"health-lock-online",
|
||||
Duration::from_secs(1),
|
||||
async { Ok(tokio::task::id()) },
|
||||
RpcCleanup::new(None, None, None),
|
||||
)
|
||||
.await
|
||||
.expect("successful RPC must return its result");
|
||||
assert_eq!(rpc_id, caller_id, "successful RPC must not allocate and schedule a separate task");
|
||||
})
|
||||
.await
|
||||
.expect("caller task must complete");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_remote_client_rpc_panic_isolated_and_cleans_up() {
|
||||
let (released_tx, released_rx) = tokio::sync::oneshot::channel();
|
||||
let caller = tokio::spawn(async move {
|
||||
let client = RemoteClient::new("http://direct-poll-panic".to_string());
|
||||
let late_failure: LateFailure = Some(Box::new(move |()| {
|
||||
Box::pin(async move {
|
||||
released_tx.send(()).expect("panic cleanup observer remains alive");
|
||||
})
|
||||
}));
|
||||
client
|
||||
.execute_rpc::<(), _>(
|
||||
"lock",
|
||||
"bucket/object",
|
||||
Duration::from_secs(1),
|
||||
async { panic!("test peer RPC panic") },
|
||||
RpcCleanup::new(None, late_failure, None),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let result = caller.await.expect("RPC panic must not escape to its caller");
|
||||
assert!(
|
||||
matches!(result, Err(LockError::Internal { .. })),
|
||||
"RPC panic remains an internal lock failure"
|
||||
);
|
||||
timeout(Duration::from_secs(1), released_rx)
|
||||
.await
|
||||
.expect("panic must trigger ambiguous-grant cleanup")
|
||||
.expect("cleanup signal");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_remote_client_timeout_keeps_channel_of_recently_serving_peer() {
|
||||
|
||||
@@ -18,6 +18,7 @@ use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use codec::{fixture_inline_pool_meta_replicas as replicas, fixture_split_inline_pool_meta_replicas as split};
|
||||
use std::path::PathBuf;
|
||||
use tokio::fs;
|
||||
use uuid::Uuid;
|
||||
|
||||
struct PhysicalFixture {
|
||||
_directories: Vec<tempfile::TempDir>,
|
||||
@@ -216,6 +217,264 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
|
||||
"all current heads, identities and complete backups must remain unchanged"
|
||||
);
|
||||
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
|
||||
state
|
||||
.ensure_write_safe("ordinary authority temporarily unavailable")
|
||||
.expect("missing ordinary authority alone must not install a permanent corruption latch");
|
||||
}
|
||||
|
||||
async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
|
||||
let fixture = fixture(3).await;
|
||||
let PoolMetaReplica::Valid {
|
||||
previous: Some(previous),
|
||||
committed: false,
|
||||
..
|
||||
} = decode_pool_meta_replica(fixture.pending.clone())
|
||||
else {
|
||||
panic!("fixture must retain its acknowledged predecessor");
|
||||
};
|
||||
let physical = disks(&fixture.store).await;
|
||||
let prepared = replicas(&fixture.pending, OffsetDateTime::now_utc()).await;
|
||||
for (index, disk) in physical[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &prepared[index])
|
||||
.await
|
||||
.expect("one pool must retain an ordinary-readable PREPARE");
|
||||
}
|
||||
(fixture, *previous)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_interrupted_first_commit_restores_proven_predecessor() {
|
||||
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let before = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(
|
||||
before[1..]
|
||||
.iter()
|
||||
.all(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
|
||||
);
|
||||
assert!(matches!(before[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect("ordinary PREPARE must authorize restoring only its acknowledged predecessor")
|
||||
);
|
||||
state
|
||||
.ensure_write_safe("first commit rollback")
|
||||
.expect("rollback must finish its transaction");
|
||||
assert_ne!(
|
||||
previous.canonical, fixture.canonical,
|
||||
"the interrupted COMMIT must not become authoritative"
|
||||
);
|
||||
fixture.canonical = previous.canonical;
|
||||
fixture.revision = previous.revision;
|
||||
assert_committed(&fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_interrupted_rollback_retains_authority_and_resumes() {
|
||||
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let pause = Arc::new(NativeApplyPause::default());
|
||||
let reached = pause.reached.notified();
|
||||
tokio::pin!(reached);
|
||||
reached.as_mut().enable();
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
let mut state = clean_state(&fixture);
|
||||
let recovery = tokio::spawn({
|
||||
let store = Arc::clone(&fixture.store);
|
||||
let pause = Arc::clone(&pause);
|
||||
async move {
|
||||
AFTER_NATIVE_POOL_APPLY
|
||||
.scope(pause, async move { repair(&store, &mut state, &fence).await })
|
||||
.await
|
||||
}
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), reached)
|
||||
.await
|
||||
.expect("real rollback should pause after one pool is durably restored");
|
||||
let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(partial[0].replica, PoolMetaReplica::Valid { committed: false, .. }));
|
||||
assert!(matches!(&partial[1].replica, PoolMetaReplica::Valid { canonical, committed: true, .. }
|
||||
if canonical == &previous.canonical));
|
||||
assert!(matches!(partial[2].replica, PoolMetaReplica::Unreadable(_)));
|
||||
recovery.abort();
|
||||
assert!(
|
||||
recovery
|
||||
.await
|
||||
.expect_err("the paused repair must be cancelled")
|
||||
.is_cancelled()
|
||||
);
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect("restart must resume from the retained ordinary PREPARE")
|
||||
);
|
||||
fixture.canonical = previous.canonical;
|
||||
fixture.revision = previous.revision;
|
||||
assert_committed(&fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_rollback_resumes_after_last_prepare_loses_read_quorum() {
|
||||
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
let inspected = inspect(&fixture.store.pools, &fence)
|
||||
.await
|
||||
.expect("physical rollback inventory");
|
||||
let replacements = plan(
|
||||
&inspected,
|
||||
&fixture.canonical,
|
||||
fixture.revision,
|
||||
&fixture.identity,
|
||||
Some(&previous.canonical),
|
||||
)
|
||||
.await
|
||||
.expect("the ordinary PREPARE must authorize its predecessor");
|
||||
for (pool, count) in [(1, 4), (2, 4), (0, 2)] {
|
||||
for (index, replacement) in replacements[pool].iter().enumerate().take(count) {
|
||||
let snapshot = &inspected[pool].snapshots[index];
|
||||
assert_eq!(
|
||||
inspected[pool].disks[index]
|
||||
.pool_meta_native_recovery(Request::Apply {
|
||||
object: PoolMetaNativeObject::PoolBin,
|
||||
expected_disk_id: snapshot.disk_id,
|
||||
expected_current_sha256: snapshot.current.as_ref().map(|raw| Sha256::digest(raw).into()),
|
||||
expected_backup_inventory_sha256: snapshot.inventory_sha256().expect("original inventory hash"),
|
||||
replacement: replacement.clone(),
|
||||
})
|
||||
.await
|
||||
.expect("partial rollback must use the real conditional publication and sync path"),
|
||||
Response::Applied
|
||||
);
|
||||
}
|
||||
}
|
||||
let partial = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(partial[0].replica, PoolMetaReplica::Unreadable(_)));
|
||||
assert!(partial[1..].iter().all(|read| matches!(&read.replica,
|
||||
PoolMetaReplica::Valid { canonical, committed: true, .. } if canonical == &previous.canonical)));
|
||||
let mut state = clean_state(&fixture);
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect("restored ordinary COMMIT must finish rollback after the last PREPARE becomes split")
|
||||
);
|
||||
fixture.canonical = previous.canonical;
|
||||
fixture.revision = previous.revision;
|
||||
assert_committed(&fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_rollback_backup_does_not_poison_reused_generation() {
|
||||
let (mut fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(repair(&fixture.store, &mut state, &fence).await.expect("initial rollback"));
|
||||
let revision = PoolMetaRevision {
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
..fixture.revision
|
||||
};
|
||||
let mut meta = previous.meta.clone();
|
||||
meta.pools[0].last_update += Duration::seconds(1);
|
||||
let canonical = encode_pool_meta_v3_envelope(&meta, revision, true, None).expect("later COMMIT");
|
||||
let pending = encode_pool_meta_v3_envelope(&meta, revision, false, Some(&previous)).expect("later PREPARE");
|
||||
let modified = OffsetDateTime::now_utc() + Duration::seconds(1);
|
||||
let split = split(&pending, &canonical, modified).await;
|
||||
let commit = replicas(&canonical, modified).await;
|
||||
let prepared = replicas(&pending, modified).await;
|
||||
let directory = Uuid::new_v4();
|
||||
let retained = snapshots(&fixture.store).await;
|
||||
for (pool_index, pool) in disks(&fixture.store).await.into_iter().enumerate() {
|
||||
for (disk_index, disk) in pool.iter().enumerate() {
|
||||
assert!(
|
||||
!retained[pool_index][disk_index].backups.is_empty(),
|
||||
"the abandoned PREPARE must remain as real on-disk evidence"
|
||||
);
|
||||
let object = object_path(disk);
|
||||
fs::write(object.join("xl.meta"), &if pool_index < 2 { &split } else { &commit }[disk_index])
|
||||
.await
|
||||
.expect("install later partial COMMIT without clearing old backups");
|
||||
fs::create_dir(object.join(directory.to_string()))
|
||||
.await
|
||||
.expect("later backup directory");
|
||||
fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &prepared[disk_index])
|
||||
.await
|
||||
.expect("later PREPARE backup");
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect("an ordinary later COMMIT must survive the abandoned sibling PREPARE")
|
||||
);
|
||||
fixture.canonical = canonical;
|
||||
fixture.revision = revision;
|
||||
assert_committed(&fixture).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_rollback_rejects_retained_commit_without_mutation() {
|
||||
let (fixture, _) = interrupted_first_commit_fixture().await;
|
||||
let physical = disks(&fixture.store).await;
|
||||
let commit = replicas(&fixture.canonical, OffsetDateTime::now_utc()).await;
|
||||
let directory = Uuid::new_v4();
|
||||
let object = object_path(&physical[0][0]);
|
||||
fs::create_dir(object.join(directory.to_string()))
|
||||
.await
|
||||
.expect("retained COMMIT directory");
|
||||
fs::write(object.join(directory.to_string()).join("xl.meta.bkp"), &commit[0])
|
||||
.await
|
||||
.expect("retain evidence that the interrupted transaction may already have committed");
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(repair(&fixture.store, &mut state, &fence).await.is_err());
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "ambiguous COMMIT evidence must remain untouched");
|
||||
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_physical_rollback_rejects_competing_prepares_without_mutation() {
|
||||
let (fixture, previous) = interrupted_first_commit_fixture().await;
|
||||
let pending = encode_pool_meta_v3_envelope(
|
||||
&previous.meta,
|
||||
PoolMetaRevision {
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
..fixture.revision
|
||||
},
|
||||
false,
|
||||
Some(&previous),
|
||||
)
|
||||
.expect("competing PREPARE must remain structurally valid");
|
||||
let physical = disks(&fixture.store).await;
|
||||
let prepared = replicas(&pending, OffsetDateTime::now_utc()).await;
|
||||
for (index, disk) in physical[1].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &prepared[index])
|
||||
.await
|
||||
.expect("second pool must expose the competing transaction through ordinary reads");
|
||||
}
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
let error = repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect_err("two distinct prepared transactions must not authorize a guessed rollback");
|
||||
assert_eq!(
|
||||
error
|
||||
.pool_metadata_failure()
|
||||
.expect("transaction conflict must be typed")
|
||||
.kind,
|
||||
crate::error::PoolMetadataFailure::RecoveryRequired
|
||||
);
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "competing transactions must remain untouched");
|
||||
assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
struct IdentityEpochFork {
|
||||
@@ -467,7 +726,7 @@ async fn pool_meta_native_physical_partial_apply_resumes_after_conditional_rejec
|
||||
let inspected = inspect(&fixture.store.pools, &fence)
|
||||
.await
|
||||
.expect("real native snapshots should be obtained");
|
||||
let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity)
|
||||
let replacements = plan(&inspected, &fixture.canonical, fixture.revision, &fixture.identity, None)
|
||||
.await
|
||||
.expect("physical recovery plan should validate");
|
||||
let snapshot = &inspected[0].snapshots[0];
|
||||
@@ -628,3 +887,559 @@ async fn pool_meta_native_physical_three_plus_one_checkpoint_converges_all_shard
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn owner_repair_no_backups_fixture() -> PhysicalFixture {
|
||||
let fixture = fixture(0).await;
|
||||
for pool in disks(&fixture.store).await {
|
||||
for disk in pool {
|
||||
let object = object_path(&disk);
|
||||
let mut entries = fs::read_dir(&object).await.expect("fixture metadata directory");
|
||||
while let Some(entry) = entries.next_entry().await.expect("fixture directory entry") {
|
||||
if entry.file_type().await.expect("fixture entry type").is_dir() {
|
||||
fs::remove_dir_all(entry.path())
|
||||
.await
|
||||
.expect("remove task-owned fixture rollback directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fixture
|
||||
}
|
||||
|
||||
async fn owner_repair_identity_heads(store: &ECStore) -> Vec<Vec<(Uuid, Bytes)>> {
|
||||
let mut all = Vec::new();
|
||||
for pool in &store.pools {
|
||||
let set = pool.get_disks_by_key(POOL_META_IDENTITY_NAME);
|
||||
let members = set
|
||||
.disks
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.expect("identity fixture disk must be online"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut heads = Vec::new();
|
||||
for disk in members {
|
||||
match disk
|
||||
.pool_meta_native_recovery(Request::InspectIdentity {})
|
||||
.await
|
||||
.expect("physical identity inspection")
|
||||
{
|
||||
Response::IdentityInspected { disk_id, raw } => heads.push((disk_id, raw)),
|
||||
_ => panic!("identity inspection must return its physical head"),
|
||||
}
|
||||
}
|
||||
all.push(heads);
|
||||
}
|
||||
all
|
||||
}
|
||||
|
||||
async fn legacy_native_owner_fixture(version: u16, stale: bool) -> (PhysicalFixture, Vec<u8>) {
|
||||
let fixture = owner_repair_no_backups_fixture().await;
|
||||
let mut meta = match decode_pool_meta_replica(fixture.canonical.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("fixture must provide valid committed pool metadata"),
|
||||
};
|
||||
meta.version = version;
|
||||
let raw = meta
|
||||
.encode_config_data_for_v2_gate(version == POOL_META_VERSION)
|
||||
.expect("legacy fixture should encode in its original version");
|
||||
assert_eq!(u16::from_le_bytes([raw[2], raw[3]]), version);
|
||||
let all = disks(&fixture.store).await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
for pool in &all {
|
||||
let heads = replicas(&raw, now).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install exact legacy head");
|
||||
}
|
||||
}
|
||||
if stale {
|
||||
let old = replicas(&raw, now - Duration::seconds(10)).await;
|
||||
fs::write(object_path(&all[0][0]).join("xl.meta"), &old[0])
|
||||
.await
|
||||
.expect("install stale same-raw legacy head");
|
||||
}
|
||||
(fixture, raw)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_v1_with_all_upgrade_gates_disabled_keeps_exact_raw() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(0, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
assert!(result.is_ok(), "fully online legacy native heal should succeed: {result:?}");
|
||||
drop(state);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for read in &reads {
|
||||
assert!(
|
||||
matches!(&read.replica, PoolMetaReplica::Valid { raw: bytes, revision, .. }
|
||||
if bytes == &raw && revision.version == POOL_META_V1_VERSION),
|
||||
"native heal must not upgrade a V1 raw payload when both upgrade gates are disabled"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_legacy_source_with_trailing_bytes_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
let (fixture, raw) = legacy_native_owner_fixture(version, false).await;
|
||||
let mut source_raw = raw.clone();
|
||||
// A second MessagePack value is readable by legacy decoding but
|
||||
// must not be copied as part of an unverified repair payload.
|
||||
source_raw.push(0xc0);
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(source_raw.clone()), PoolMetaReplica::Valid { raw, .. } if raw == source_raw),
|
||||
"ordinary legacy reads must remain compatible with trailing bytes"
|
||||
);
|
||||
let all_disks = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&source_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in all_disks[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install readable legacy source with trailing bytes");
|
||||
}
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await
|
||||
.expect("legacy semantic selection must remain readable");
|
||||
assert_eq!(selection.legacy_raw.as_deref(), Some(source_raw.as_slice()));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("legacy native heal must terminate");
|
||||
drop(state);
|
||||
assert!(result.is_err(), "version {version}: unparsed trailing source bytes must not be republished");
|
||||
assert_eq!(snapshots(&fixture.store).await, before, "version {version}: no pool.bin head or backup may change");
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"version {version}: no identity shard may change"
|
||||
);
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &source_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw: target, .. } if target == &raw));
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_mixed_v1_v2_preserves_observed_format_floor() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for first_version in [POOL_META_V1_VERSION, POOL_META_VERSION] {
|
||||
for target_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let mut meta = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { meta, committed: true, .. } => meta,
|
||||
_ => panic!("initial V1 metadata must decode"),
|
||||
};
|
||||
meta.version = POOL_META_VERSION;
|
||||
let raw_v2 = meta.encode_config_data_for_v2_gate(true).expect("equivalent V2 payload must encode");
|
||||
assert_eq!(u16::from_le_bytes([raw_v1[2], raw_v1[3]]), POOL_META_V1_VERSION);
|
||||
assert_eq!(u16::from_le_bytes([raw_v2[2], raw_v2[3]]), POOL_META_VERSION);
|
||||
let semantic_v1 = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V1 payload must remain valid"),
|
||||
};
|
||||
let semantic_v2 = match decode_pool_meta_replica(raw_v2.clone()) {
|
||||
PoolMetaReplica::Valid { canonical, .. } => canonical,
|
||||
_ => panic!("V2 payload must remain valid"),
|
||||
};
|
||||
assert_eq!(semantic_v1, semantic_v2, "the two formats must carry equivalent metadata");
|
||||
let other_version = if first_version == POOL_META_V1_VERSION { POOL_META_VERSION } else { POOL_META_V1_VERSION };
|
||||
// Pool two follows pool zero; pools zero and one cover both
|
||||
// orderings and both target formats in separate runs.
|
||||
let versions = [first_version, other_version, first_version];
|
||||
let physical = disks(&fixture.store).await;
|
||||
let timestamp = OffsetDateTime::now_utc();
|
||||
for (pool_index, pool) in physical.iter().enumerate() {
|
||||
let raw = if versions[pool_index] == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
let heads = replicas(raw, timestamp).await;
|
||||
for (index, disk) in pool.iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index]).await.expect("install mixed legacy pool head");
|
||||
}
|
||||
}
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let selection = load_pool_meta_replicas(fixture.store.pools.clone(), true)
|
||||
.await.expect("equivalent legacy pools should be selectable");
|
||||
assert_eq!(selection.revision.version, POOL_META_VERSION, "the observed V2 format floor must be retained");
|
||||
assert_eq!(selection.canonical.as_deref(), Some(semantic_v2.as_slice()));
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(target_pool, guard, &mut state))
|
||||
.await.expect("mixed legacy heal must terminate")
|
||||
.expect("equivalent legacy metadata should heal without downgrading the format floor");
|
||||
drop(state);
|
||||
let after = snapshots(&fixture.store).await;
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
for (pool_index, read) in reads.iter().enumerate() {
|
||||
let expected_version = if pool_index == target_pool { POOL_META_VERSION } else { versions[pool_index] };
|
||||
let expected_raw = if expected_version == POOL_META_V1_VERSION { &raw_v1 } else { &raw_v2 };
|
||||
assert!(matches!(&read.replica,
|
||||
PoolMetaReplica::Valid { raw, canonical, revision, committed: true, .. }
|
||||
if raw == expected_raw && canonical == &semantic_v2 && revision.version == expected_version
|
||||
), "pool {pool_index}, first={first_version}, target={target_pool}: preserve equivalent payload and expected legacy version");
|
||||
for (index, snapshot) in after[pool_index].iter().enumerate() {
|
||||
codec::validate_inline_pool_meta_shard(snapshot.current.as_ref().expect("physical head"), index, expected_raw)
|
||||
.await.expect("every physical shard must match its selected legacy payload");
|
||||
}
|
||||
if pool_index != target_pool {
|
||||
assert_eq!(after[pool_index], before[pool_index], "healing one pool must not modify other pools");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_and_v3_prepare_in_either_pool_order() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
for prepare_pool in [0, 1] {
|
||||
let (fixture, raw_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let (legacy_meta, legacy_canonical) = match decode_pool_meta_replica(raw_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta,
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} if revision.version == POOL_META_V1_VERSION => (meta, canonical),
|
||||
_ => panic!("baseline V1 metadata must be committed and readable"),
|
||||
};
|
||||
let previous = PoolMetaCommittedCandidate {
|
||||
canonical: legacy_canonical,
|
||||
meta: legacy_meta.clone(),
|
||||
revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
|
||||
};
|
||||
let mut next = legacy_meta;
|
||||
next.version = POOL_META_GENERATION_VERSION;
|
||||
next.pools[0].last_update += Duration::seconds(1);
|
||||
let revision = PoolMetaRevision {
|
||||
version: POOL_META_GENERATION_VERSION,
|
||||
cluster_id: fixture.revision.cluster_id,
|
||||
epoch: fixture.revision.epoch,
|
||||
generation: 1,
|
||||
transaction_id: Some(Uuid::new_v4()),
|
||||
};
|
||||
let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous))
|
||||
.expect("V3 PREPARE with V1 predecessor must encode");
|
||||
assert!(matches!(decode_pool_meta_replica(pending.clone()),
|
||||
PoolMetaReplica::Valid { committed: false, revision: actual, .. } if actual == revision));
|
||||
let heads = replicas(&pending, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
let physical = disks(&fixture.store).await;
|
||||
for (index, disk) in physical[prepare_pool].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &heads[index])
|
||||
.await
|
||||
.expect("install task-owned V3 PREPARE evidence");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[prepare_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: false, revision: actual, .. }
|
||||
if raw == &pending && *actual == revision));
|
||||
let other_pool = 1 - prepare_pool;
|
||||
assert!(matches!(&reads[other_pool].replica,
|
||||
PoolMetaReplica::Valid { raw, committed: true, revision: actual, .. }
|
||||
if raw == &raw_v1 && actual.version == POOL_META_V1_VERSION));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture
|
||||
.store
|
||||
.heal_pool_meta_replica_under_fence(prepare_pool, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("mixed V1/V3 PREPARE heal must terminate");
|
||||
drop(state);
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"pool {prepare_pool}: native heal must not overwrite a V3 PREPARE with legacy data"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"pool {prepare_pool}: rejection must retain all physical head and backup evidence"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"pool {prepare_pool}: rejection must not mutate identity evidence"
|
||||
);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_v1_legacy_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode source");
|
||||
assert!(old_reader.dont_save, "the old reader would treat this source as permanently do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source = match decode_pool_meta_replica(dangerous_raw.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("current reader must recognize the legacy source"),
|
||||
};
|
||||
assert!(matches!(decode_pool_meta_replica(safe_v1.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source && revision.version == POOL_META_V1_VERSION));
|
||||
let physical = disks(&fixture.store).await;
|
||||
let source_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[0].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &source_heads[index])
|
||||
.await
|
||||
.expect("install legacy source on pool zero");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a source whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve every original head and backup"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1),
|
||||
"the dangerous source bytes must not be copied into the target pool"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_heal_rejects_target_v1_dont_save_true_without_writes() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_POOL_META_V2_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_WRITE, Some("false")),
|
||||
(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("false")),
|
||||
],
|
||||
async {
|
||||
let (fixture, safe_v1) = legacy_native_owner_fixture(POOL_META_V1_VERSION, false).await;
|
||||
let meta = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
meta, committed: true, ..
|
||||
} => meta,
|
||||
_ => panic!("baseline V1 metadata must decode"),
|
||||
};
|
||||
let statuses = meta
|
||||
.pools
|
||||
.iter()
|
||||
.map(|pool| {
|
||||
assert!(pool.decommission.is_none(), "this regression fixture does not inject decommission state");
|
||||
LegacyPoolStatus {
|
||||
id: pool.id,
|
||||
cmd_line: pool.cmd_line.clone(),
|
||||
last_update: pool.last_update,
|
||||
decommission: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: statuses,
|
||||
dont_save: true,
|
||||
};
|
||||
let mut dangerous_raw = Vec::new();
|
||||
dangerous_raw.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
dangerous_raw.extend_from_slice(&rmp_serde::to_vec(&legacy).expect("legacy three-field tuple must serialize"));
|
||||
let old_reader: LegacyPoolMeta =
|
||||
rmp_serde::from_slice(&dangerous_raw[4..]).expect("legacy reader must decode target");
|
||||
assert!(old_reader.dont_save, "the old reader interprets the target as do-not-save");
|
||||
assert_eq!(rmp::decode::read_array_len(&mut &dangerous_raw[4..]).expect("tuple length"), 3);
|
||||
let source_canonical = match decode_pool_meta_replica(safe_v1.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: true,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revision.version, POOL_META_V1_VERSION);
|
||||
canonical
|
||||
}
|
||||
_ => panic!("safe pool-zero source must decode"),
|
||||
};
|
||||
assert!(
|
||||
matches!(decode_pool_meta_replica(dangerous_raw.clone()),
|
||||
PoolMetaReplica::Valid { canonical, revision, committed: true, .. }
|
||||
if canonical == source_canonical && revision.version == POOL_META_V1_VERSION),
|
||||
"the dangerous target must remain semantically equivalent to the safe source"
|
||||
);
|
||||
let physical = disks(&fixture.store).await;
|
||||
let target_heads = replicas(&dangerous_raw, OffsetDateTime::now_utc() + Duration::seconds(1)).await;
|
||||
for (index, disk) in physical[1].iter().enumerate() {
|
||||
fs::write(object_path(disk).join("xl.meta"), &target_heads[index])
|
||||
.await
|
||||
.expect("install legacy dangerous target head");
|
||||
}
|
||||
let reads = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&reads[0].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &safe_v1));
|
||||
assert!(matches!(&reads[1].replica, PoolMetaReplica::Valid { raw, committed: true, .. } if raw == &dangerous_raw));
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let identities = owner_repair_identity_heads(&fixture.store).await;
|
||||
let guard = owner(&fixture.store).await;
|
||||
let mut state = fixture.store.pool_meta_save_gate.lock().await;
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(30),
|
||||
fixture.store.heal_pool_meta_replica_under_fence(1, guard, &mut state),
|
||||
)
|
||||
.await
|
||||
.expect("unsafe legacy target heal must terminate");
|
||||
drop(state);
|
||||
let error = result.expect_err("native heal must reject a target whose old-reader dont_save is true");
|
||||
assert!(
|
||||
error.to_string().contains("dont_save"),
|
||||
"rejection must identify the unsafe legacy field: {error}"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
"rejection must preserve all physical heads and backups"
|
||||
);
|
||||
assert_eq!(
|
||||
owner_repair_identity_heads(&fixture.store).await,
|
||||
identities,
|
||||
"rejection must not write identity shards"
|
||||
);
|
||||
let after = read_pool_meta_replicas(fixture.store.pools.clone(), true).await;
|
||||
assert!(matches!(&after[0].replica, PoolMetaReplica::Valid { raw, .. } if raw == &safe_v1));
|
||||
assert!(
|
||||
matches!(&after[1].replica, PoolMetaReplica::Valid { raw, .. } if raw == &dangerous_raw),
|
||||
"unsafe target evidence must not be overwritten by the safe source"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ pub(super) struct NativeApplyPause {
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(super) static AFTER_NATIVE_APPLY: Arc<NativeApplyPause>;
|
||||
pub(super) static AFTER_NATIVE_POOL_APPLY: Arc<NativeApplyPause>;
|
||||
}
|
||||
|
||||
struct InspectedPool {
|
||||
@@ -168,6 +169,7 @@ async fn plan(
|
||||
canonical: &[u8],
|
||||
revision: PoolMetaRevision,
|
||||
identity: &[u8],
|
||||
rollback: Option<&[u8]>,
|
||||
) -> Result<Vec<Vec<Bytes>>> {
|
||||
let mut pending = Vec::<Vec<u8>>::new();
|
||||
let mut predecessors = Vec::<Vec<u8>>::new();
|
||||
@@ -223,23 +225,83 @@ async fn plan(
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut obsolete = Vec::<Vec<u8>>::new();
|
||||
for payload in &decoded {
|
||||
match decode_pool_meta_replica(payload.clone()) {
|
||||
PoolMetaReplica::Valid {
|
||||
// The last ordinary PREPARE can itself become split during rollback. In
|
||||
// that case another pool already exposes the restored ordinary COMMIT.
|
||||
// Its authority permits continuing the rollback, never promoting a child
|
||||
// transaction reconstructed from disk evidence.
|
||||
let mut continuation: Option<(Vec<u8>, PoolMetaRevision)> = None;
|
||||
if rollback.is_none() {
|
||||
let mut outstanding = Vec::new();
|
||||
for pool in inspected {
|
||||
for (index, snapshot) in pool.snapshots.iter().enumerate() {
|
||||
let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
|
||||
if codec::validate_inline_pool_meta_shard(raw, index, canonical).await.is_err() {
|
||||
outstanding.push((index, raw));
|
||||
}
|
||||
}
|
||||
}
|
||||
for payload in &decoded {
|
||||
if let PoolMetaReplica::Valid {
|
||||
canonical: candidate,
|
||||
revision: candidate_revision,
|
||||
committed: false,
|
||||
previous: Some(previous),
|
||||
..
|
||||
} if candidate_revision == revision && candidate == canonical => {
|
||||
if !pending.contains(payload) {
|
||||
pending.push(payload.clone());
|
||||
} = decode_pool_meta_replica(payload.clone())
|
||||
&& previous.revision == revision
|
||||
&& previous.canonical == canonical
|
||||
{
|
||||
let mut matches = true;
|
||||
for (index, raw) in &outstanding {
|
||||
matches &= codec::validate_inline_pool_meta_shard(raw, *index, payload).await.is_ok()
|
||||
|| codec::validate_inline_pool_meta_shard(raw, *index, &candidate).await.is_ok();
|
||||
}
|
||||
if !predecessors.contains(&previous.canonical) {
|
||||
predecessors.push(previous.canonical);
|
||||
if matches {
|
||||
if !outstanding.is_empty()
|
||||
&& continuation.as_ref().is_some_and(|(selected, selected_revision)| {
|
||||
selected != &candidate || *selected_revision != candidate_revision
|
||||
})
|
||||
{
|
||||
return Err(known_validation(invalid(), "native_transaction_validation"));
|
||||
}
|
||||
continuation.get_or_insert((candidate, candidate_revision));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let (canonical, revision, rollback) = match &continuation {
|
||||
Some((transaction, next_revision)) => (transaction.as_slice(), *next_revision, Some(canonical)),
|
||||
None => (canonical, revision, rollback),
|
||||
};
|
||||
for payload in &decoded {
|
||||
if let PoolMetaReplica::Valid {
|
||||
canonical: candidate,
|
||||
revision: candidate_revision,
|
||||
committed: false,
|
||||
previous: Some(previous),
|
||||
..
|
||||
} = decode_pool_meta_replica(payload.clone())
|
||||
&& candidate_revision == revision
|
||||
&& candidate == canonical
|
||||
{
|
||||
pending.push(payload.clone());
|
||||
if !predecessors.contains(&previous.canonical) {
|
||||
predecessors.push(previous.canonical);
|
||||
}
|
||||
}
|
||||
}
|
||||
if pending.len() > 1 {
|
||||
return Err(known_validation(
|
||||
Error::other("native V3 prepare records disagree on the predecessor"),
|
||||
"native_transaction_validation",
|
||||
));
|
||||
}
|
||||
let mut obsolete = Vec::<Vec<u8>>::new();
|
||||
for payload in &decoded {
|
||||
if pending.contains(payload) {
|
||||
continue;
|
||||
}
|
||||
match decode_pool_meta_replica(payload.clone()) {
|
||||
PoolMetaReplica::Valid { revision: older, .. }
|
||||
if older.is_generation_protocol()
|
||||
&& older.cluster_id == revision.cluster_id
|
||||
@@ -251,6 +313,23 @@ async fn plan(
|
||||
// fully reconstructed and bound into the native CAS.
|
||||
obsolete.push(payload.clone());
|
||||
}
|
||||
PoolMetaReplica::Valid {
|
||||
revision: abandoned,
|
||||
committed: false,
|
||||
previous: Some(previous),
|
||||
..
|
||||
} if abandoned.is_generation_protocol()
|
||||
&& abandoned.cluster_id == revision.cluster_id
|
||||
&& abandoned.epoch == revision.epoch
|
||||
&& abandoned.generation == revision.generation
|
||||
&& abandoned.transaction_id != revision.transaction_id
|
||||
&& predecessors.contains(&previous.canonical) =>
|
||||
{
|
||||
// A rollback retains the failed PREPARE as evidence. A later
|
||||
// transaction may reuse that generation, but only a backup of
|
||||
// an uncommitted sibling with the same predecessor is obsolete.
|
||||
obsolete.push(payload.clone());
|
||||
}
|
||||
PoolMetaReplica::Valid {
|
||||
canonical: candidate,
|
||||
revision: candidate_revision,
|
||||
@@ -272,13 +351,7 @@ async fn plan(
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if pending.len() > 1 {
|
||||
return Err(known_validation(
|
||||
Error::other("native V3 prepare records disagree on the predecessor"),
|
||||
"native_transaction_validation",
|
||||
));
|
||||
}
|
||||
if pending.is_empty() {
|
||||
if pending.is_empty() || rollback.is_some_and(|previous| !predecessors.iter().any(|candidate| candidate == previous)) {
|
||||
return Err(invalid());
|
||||
}
|
||||
for payload in &decoded {
|
||||
@@ -294,13 +367,25 @@ async fn plan(
|
||||
for (index, snapshot) in pool.snapshots.iter().enumerate() {
|
||||
let raw = snapshot.current.as_ref().ok_or_else(invalid)?;
|
||||
let mut matched = false;
|
||||
for payload in std::iter::once(canonical).chain(pending.iter().map(Vec::as_slice)) {
|
||||
for payload in std::iter::once(canonical)
|
||||
.chain(pending.iter().map(Vec::as_slice))
|
||||
.chain(rollback)
|
||||
{
|
||||
matched |= codec::validate_inline_pool_meta_shard(raw, index, payload).await.is_ok();
|
||||
}
|
||||
if !matched {
|
||||
return Err(invalid());
|
||||
}
|
||||
for backup in &snapshot.backups {
|
||||
if rollback.is_some()
|
||||
&& codec::validate_inline_pool_meta_shard(&backup.data, index, canonical)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
// A retained COMMIT might have been authoritative before
|
||||
// this interruption. Never roll it back from PREPARE alone.
|
||||
return Err(invalid());
|
||||
}
|
||||
let mut matched = false;
|
||||
for payload in std::iter::once(canonical).chain(
|
||||
pending
|
||||
@@ -323,7 +408,7 @@ async fn plan(
|
||||
let mut replacements = Vec::with_capacity(inspected.len());
|
||||
for pool in inspected {
|
||||
replacements.push(
|
||||
codec::encode_inline_pool_meta_replicas(¤t(pool)?, canonical, modified)
|
||||
codec::encode_inline_pool_meta_replicas(¤t(pool)?, rollback.unwrap_or(canonical), modified)
|
||||
.await
|
||||
.map_err(|error| codec_validation(error, "native_metadata_validation"))?,
|
||||
);
|
||||
@@ -338,10 +423,12 @@ pub(super) async fn repair(
|
||||
) -> Result<bool> {
|
||||
fence.ensure_held()?;
|
||||
let reads = read_pool_meta_replicas(store.pools.clone(), true).await;
|
||||
if !reads
|
||||
let unreadable = reads
|
||||
.iter()
|
||||
.any(|read| matches!(read.replica, PoolMetaReplica::Unreadable(_)))
|
||||
{
|
||||
.enumerate()
|
||||
.filter_map(|(index, read)| matches!(read.replica, PoolMetaReplica::Unreadable(_)).then_some(index))
|
||||
.collect::<Vec<_>>();
|
||||
if unreadable.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
state.ensure_write_safe("native pool metadata recovery")?;
|
||||
@@ -360,8 +447,9 @@ pub(super) async fn repair(
|
||||
{
|
||||
return Err(invalid());
|
||||
}
|
||||
// Only an ordinary, successfully read COMMIT grants authority. Rebuilding
|
||||
// two erasure shards below proves integrity, never a commit decision.
|
||||
// Ordinary reads grant authority: a COMMIT selects its own state, while a
|
||||
// PREPARE can select only its acknowledged predecessor. Reconstruction
|
||||
// below proves integrity and never promotes an interrupted transaction.
|
||||
let ordinary_commits = reads
|
||||
.iter()
|
||||
.filter_map(|read| match &read.replica {
|
||||
@@ -375,7 +463,22 @@ pub(super) async fn repair(
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if ordinary_commits.is_empty() {
|
||||
let ordinary_prepares = reads
|
||||
.iter()
|
||||
.filter_map(|read| match &read.replica {
|
||||
PoolMetaReplica::Valid {
|
||||
canonical,
|
||||
revision,
|
||||
committed: false,
|
||||
previous: Some(previous),
|
||||
..
|
||||
} if revision.is_generation_protocol() && previous.revision.is_generation_protocol() => {
|
||||
Some((canonical.clone(), *revision, previous.canonical.clone()))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
|
||||
return Err(invalid());
|
||||
}
|
||||
let selection = select_pool_meta_replica_reads(reads)
|
||||
@@ -395,10 +498,20 @@ pub(super) async fn repair(
|
||||
return Err(invalid());
|
||||
}
|
||||
let canonical = selection.canonical.as_ref().ok_or_else(invalid)?;
|
||||
let mut rollback = None;
|
||||
for (pending, revision, previous) in &ordinary_prepares {
|
||||
if previous == canonical {
|
||||
if rollback.is_some_and(|candidate| candidate != (pending, *revision)) {
|
||||
return Err(latch_validation(state, known_validation(invalid(), "native_transaction_validation")));
|
||||
}
|
||||
rollback = Some((pending, *revision));
|
||||
}
|
||||
}
|
||||
if !selection.revision.is_generation_protocol()
|
||||
|| !ordinary_commits
|
||||
.iter()
|
||||
.any(|(raw, revision)| raw == canonical && *revision == selection.revision)
|
||||
|| (rollback.is_none()
|
||||
&& !ordinary_commits
|
||||
.iter()
|
||||
.any(|(raw, revision)| raw == canonical && *revision == selection.revision))
|
||||
{
|
||||
return Err(invalid());
|
||||
}
|
||||
@@ -445,13 +558,18 @@ pub(super) async fn repair(
|
||||
.await
|
||||
.map_err(|error| latch_validation(state, error))?;
|
||||
let inspected = inspect(&store.pools, fence).await?;
|
||||
let replacements = plan(&inspected, canonical, selection.revision, &identity_bytes)
|
||||
let (transaction, revision) = rollback.unwrap_or((canonical, selection.revision));
|
||||
let replacements = plan(&inspected, transaction, revision, &identity_bytes, rollback.map(|_| canonical.as_slice()))
|
||||
.await
|
||||
.map_err(|error| latch_validation(state, error))?;
|
||||
fence.ensure_held()?;
|
||||
let mut arm = state.arm_transaction();
|
||||
arm.phase = Some("native_repair_apply");
|
||||
for (pool, replacements) in inspected.iter().zip(replacements) {
|
||||
let mut apply_order = inspected.iter().zip(replacements).enumerate().collect::<Vec<_>>();
|
||||
// Preserve ordinary authority until all unreadable pools are repaired.
|
||||
// Cancellation can then resume from a PREPARE or a restored COMMIT.
|
||||
apply_order.sort_by_key(|(index, _)| !unreadable.contains(index));
|
||||
for (_, (pool, replacements)) in apply_order {
|
||||
for ((disk, snapshot), replacement) in pool.disks.iter().zip(&pool.snapshots).zip(replacements) {
|
||||
fence.ensure_held()?;
|
||||
let request = Request::Apply {
|
||||
@@ -466,6 +584,11 @@ pub(super) async fn repair(
|
||||
_ => return Err(invalid()),
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
if let Ok(pause) = AFTER_NATIVE_POOL_APPLY.try_with(Arc::clone) {
|
||||
pause.reached.notify_one();
|
||||
pause.resume.notified().await;
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
if let Ok(pause) = AFTER_NATIVE_APPLY.try_with(Arc::clone) {
|
||||
@@ -627,7 +750,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn pool_meta_native_plan_restores_physical_split_with_fresh_common_identity() {
|
||||
let f = fixture().await;
|
||||
let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity).await.unwrap();
|
||||
let repaired = plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.unwrap();
|
||||
assert_eq!(repaired.len(), 4);
|
||||
for pool in repaired {
|
||||
assert_eq!(
|
||||
@@ -648,7 +771,7 @@ mod tests {
|
||||
let mut f = fixture().await;
|
||||
let wrong = replicas(b"different identity", f.modified).await;
|
||||
f.pools[0].snapshots[0].identity = Some(wrong[0].clone());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -671,10 +794,10 @@ mod tests {
|
||||
.unwrap();
|
||||
let wrong = replicas(&higher, f.modified + Duration::seconds(2)).await;
|
||||
f.pools[0].snapshots[0].current = Some(wrong[0].clone());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
f.pools[0].snapshots[0].current = Some(replicas(&f.pending, f.modified).await[0].clone());
|
||||
f.pools[0].snapshots[0].backups[0].data = wrong[0].clone();
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -717,17 +840,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
assert!(older_pending != f.pending);
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_ok());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
|
||||
let mut f = fixture().await;
|
||||
let before = f.pools[1].snapshots[2].current.take();
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
f.pools[1].snapshots[2].current = before;
|
||||
f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity).await.is_err());
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -94,7 +94,6 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
#[cfg(test)]
|
||||
use std::io::Cursor;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
@@ -3720,33 +3719,45 @@ fn is_decommission_copy_quorum_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyQuorumFailure>(err).is_some()
|
||||
}
|
||||
|
||||
/// The storage error of a version copy that kept timing out on a namespace
|
||||
/// lock through its in-place retries, kept typed so the entry failure policy
|
||||
/// defers it.
|
||||
/// A copy whose object namespace lock remained unavailable through its
|
||||
/// in-place retries, retained as a typed deferral.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
struct DecommissionCopyLockTimeout(#[source] Error);
|
||||
struct DecommissionCopyLockUnavailable(#[source] Error);
|
||||
|
||||
fn is_decommission_copy_lock_timeout_error(err: &Error) -> bool {
|
||||
let err = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
matches!(err, Error::Lock(rustfs_lock::LockError::Timeout { .. }))
|
||||
fn is_decommission_copy_lock_unavailable_error(err: &Error, bucket: &str, object: &str) -> bool {
|
||||
let source = data_movement::data_movement_stage_source(err).unwrap_or(err);
|
||||
match source {
|
||||
Error::Lock(rustfs_lock::LockError::Timeout { .. }) => true,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: lock_bucket,
|
||||
object: lock_object,
|
||||
..
|
||||
} => {
|
||||
data_movement::data_movement_error_is_stage(err, "prepare_put_object")
|
||||
&& lock_bucket == bucket
|
||||
&& lock_object == object
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// A version copy that timed out acquiring a namespace lock leaves the source
|
||||
/// A version copy that could not acquire its namespace lock leaves the source
|
||||
/// in place; the next attempt copies the version again or finds an equivalent
|
||||
/// committed target. The lock is held by another mutation of the key or, when
|
||||
/// this node restarted mid-copy, by its previous incarnation until that lock's
|
||||
/// lease expires on the peers. Either clears by waiting, so the entry is
|
||||
/// deferred instead of being counted as a permanent per-object failure.
|
||||
fn decommission_copy_lock_timeout_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
fn decommission_copy_lock_unavailable_deferral(stage: &str, bucket: &str, object: &str, err: Error) -> Error {
|
||||
data_movement::data_movement_context_error(
|
||||
format!("decommission {stage} timed out on a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockTimeout(err),
|
||||
format!("decommission {stage} could not acquire a namespace lock for {bucket}/{object}: {err}"),
|
||||
DecommissionCopyLockUnavailable(err),
|
||||
)
|
||||
}
|
||||
|
||||
fn is_decommission_copy_lock_timeout_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(err).is_some()
|
||||
fn is_decommission_copy_lock_unavailable_deferral(err: &Error) -> bool {
|
||||
data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(err).is_some()
|
||||
}
|
||||
|
||||
/// The deferral of a version copy whose in-place retries ended on `error`, if
|
||||
@@ -3756,8 +3767,8 @@ fn take_decommission_copy_deferral(stage: &str, bucket: &str, object: &str, erro
|
||||
return Some(decommission_copy_quorum_deferral(stage, bucket, object, err));
|
||||
}
|
||||
error
|
||||
.take_if(|err| is_decommission_copy_lock_timeout_error(err))
|
||||
.map(|err| decommission_copy_lock_timeout_deferral(stage, bucket, object, err))
|
||||
.take_if(|err| is_decommission_copy_lock_unavailable_error(err, bucket, object))
|
||||
.map(|err| decommission_copy_lock_unavailable_deferral(stage, bucket, object, err))
|
||||
}
|
||||
|
||||
fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
|
||||
@@ -5822,13 +5833,13 @@ enum DecommissionEntryOutcome {
|
||||
/// entry is done and must not be counted as a failure or escalated. A quorum
|
||||
/// failure of the source cleanup is deferred too: the copied source is still
|
||||
/// in place and the next attempt cleans it up again. So is a version copy that
|
||||
/// lost drive quorum or timed out on a namespace lock through its in-place
|
||||
/// lost drive quorum or could not acquire an object namespace lock through its in-place
|
||||
/// retries: the source still holds the version and the next attempt copies it
|
||||
/// again. Every other failure stays terminal.
|
||||
fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
|
||||
if is_decommission_source_cleanup_retryable_error(&err)
|
||||
|| is_decommission_copy_quorum_deferral(&err)
|
||||
|| is_decommission_copy_lock_timeout_deferral(&err)
|
||||
|| is_decommission_copy_lock_unavailable_deferral(&err)
|
||||
{
|
||||
return Ok(DecommissionEntryOutcome::Deferred(err));
|
||||
}
|
||||
@@ -6827,11 +6838,47 @@ impl Drop for PoolMetaTransactionArm {
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_legacy_pool_meta_republish_safe(data: &[u8]) -> Result<()> {
|
||||
if data.len() >= 4
|
||||
&& LittleEndian::read_u16(&data[2..4]) == POOL_META_V1_VERSION
|
||||
// The selection already validates the payload. The runtime-only
|
||||
// flag can only appear in the legacy three-field sequence or map.
|
||||
&& rmp::decode::read_array_len(&mut &data[4..])
|
||||
.or_else(|_| rmp::decode::read_map_len(&mut &data[4..]))
|
||||
.ok()
|
||||
== Some(3)
|
||||
&& let Ok(legacy) = rmp_serde::from_slice::<LegacyPoolMeta>(&data[4..])
|
||||
&& legacy.dont_save
|
||||
{
|
||||
return Err(Error::other("pool metadata legacy source contains dont_save=true"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Ordinary legacy reads retain their compatibility behavior, but bytes that
|
||||
// will be copied to another pool must not contain an unparsed extension.
|
||||
fn ensure_legacy_pool_meta_payload_complete(data: &[u8]) -> Result<()> {
|
||||
let payload = data
|
||||
.get(4..)
|
||||
.ok_or_else(|| Error::other("pool metadata legacy source is too short"))?;
|
||||
let mut decoder = rmp_serde::Deserializer::new(Cursor::new(payload));
|
||||
serde::de::IgnoredAny::deserialize(&mut decoder).map_err(|_| Error::other("pool metadata legacy source is invalid"))?;
|
||||
let payload_len = u64::try_from(payload.len()).map_err(|_| Error::other("pool metadata legacy source is too large"))?;
|
||||
if decoder.get_ref().position() != payload_len {
|
||||
return Err(Error::other("pool metadata legacy source has trailing bytes"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PoolMetaSelection {
|
||||
meta: PoolMeta,
|
||||
revision: PoolMetaRevision,
|
||||
canonical: Option<Vec<u8>>,
|
||||
// Legacy semantic comparison canonicalizes to V2; a heal must instead
|
||||
// preserve verified raw bytes at the highest observed format floor.
|
||||
legacy_raw: Option<Vec<u8>>,
|
||||
legacy_has_unsafe_flag: bool,
|
||||
replica_state: PoolMetaReplicaState,
|
||||
cas_tokens: Vec<PoolMetaCasToken>,
|
||||
absent: bool,
|
||||
@@ -7222,6 +7269,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: PoolMeta::default(),
|
||||
revision: PoolMetaRevision::legacy(0),
|
||||
canonical: None,
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair: false,
|
||||
repair_write_safe: true,
|
||||
@@ -7276,6 +7325,8 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
meta: selected.meta.clone(),
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical.clone()),
|
||||
legacy_raw: None,
|
||||
legacy_has_unsafe_flag: false,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -7289,6 +7340,10 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
|
||||
// Legacy V1/V2 has no durable generation. Pool zero remains the commit
|
||||
// record; divergent backups without it are ambiguous and fail closed.
|
||||
let legacy_has_unsafe_flag = committed
|
||||
.iter()
|
||||
.any(|(_, raw, _)| ensure_legacy_pool_meta_republish_safe(raw).is_err());
|
||||
let mut floor_raw = None;
|
||||
let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
|
||||
for (idx, raw, candidate) in committed {
|
||||
if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
|
||||
@@ -7302,19 +7357,36 @@ fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<Poo
|
||||
}
|
||||
} else {
|
||||
needs_repair |= selected_raw != &raw;
|
||||
if floor_raw.is_none()
|
||||
&& candidate.revision.version == observed_version
|
||||
&& raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(raw);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
selected = Some((idx, raw, candidate));
|
||||
}
|
||||
}
|
||||
let (_, _, mut selected) =
|
||||
let (_, selected_raw, mut selected) =
|
||||
selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
|
||||
if selected.revision.version == observed_version
|
||||
&& selected_raw
|
||||
.get(2..4)
|
||||
.is_some_and(|version| LittleEndian::read_u16(version) == observed_version)
|
||||
{
|
||||
floor_raw = Some(selected_raw);
|
||||
}
|
||||
selected.meta.version = observed_version;
|
||||
selected.revision.version = observed_version;
|
||||
Ok(PoolMetaSelection {
|
||||
meta: selected.meta,
|
||||
revision: selected.revision,
|
||||
canonical: Some(selected.canonical),
|
||||
legacy_raw: floor_raw,
|
||||
legacy_has_unsafe_flag,
|
||||
replica_state: PoolMetaReplicaState {
|
||||
needs_repair,
|
||||
repair_write_safe,
|
||||
@@ -8308,8 +8380,9 @@ where
|
||||
})
|
||||
});
|
||||
let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
|
||||
let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
|
||||
if !may_have_mutated {
|
||||
let write_not_started = matches!(&result, Err(Error::Io(inner)) if inner.get_ref().is_some_and(|source| source.is::<crate::set_disk::SystemMetadataWriteNotStarted>()));
|
||||
let may_have_mutated = !write_not_started && !matches!(&result, Err(Error::PreconditionFailed));
|
||||
if matches!(&result, Err(Error::PreconditionFailed)) {
|
||||
record_pool_meta_stale_write_rejection(phase);
|
||||
}
|
||||
let mut source = None;
|
||||
@@ -8320,6 +8393,8 @@ where
|
||||
source = Some(Arc::clone(&original));
|
||||
if matches!(original.as_ref(), Error::PreconditionFailed) {
|
||||
Err(Error::PreconditionFailed)
|
||||
} else if write_not_started {
|
||||
Err(Error::other(original))
|
||||
} else {
|
||||
Err(Error::other(pool_metadata_error(
|
||||
crate::error::PoolMetadataFailure::TransactionUnknown,
|
||||
@@ -14534,6 +14609,23 @@ impl ECStore {
|
||||
.canonical
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no committed source"))?;
|
||||
if selection.generation_protocol_observed && !selection.revision.is_generation_protocol() {
|
||||
return Err(Error::other("pool metadata heal requires recovery of the observed generation protocol"));
|
||||
}
|
||||
if selection.legacy_has_unsafe_flag {
|
||||
return Err(Error::other("pool metadata legacy replica contains dont_save=true"));
|
||||
}
|
||||
let repair_data = if selection.revision.is_generation_protocol() {
|
||||
canonical.clone()
|
||||
} else {
|
||||
let raw = selection
|
||||
.legacy_raw
|
||||
.clone()
|
||||
.ok_or_else(|| Error::other("pool metadata heal cannot prove the legacy format floor"))?;
|
||||
ensure_legacy_pool_meta_payload_complete(&raw)?;
|
||||
raw
|
||||
};
|
||||
ensure_legacy_pool_meta_republish_safe(&repair_data)?;
|
||||
let pool = self
|
||||
.pools
|
||||
.get(pool_index)
|
||||
@@ -14544,10 +14636,10 @@ impl ECStore {
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| Error::other("pool metadata heal has no target revision"))?;
|
||||
let mut arm = state.arm_transaction();
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, canonical.clone(), token, &fence, "native_heal", &mut arm)
|
||||
save_pool_meta_object_cas(pool.clone(), POOL_META_NAME, repair_data.clone(), token, &fence, "native_heal", &mut arm)
|
||||
.await?;
|
||||
let (confirmed, _) = read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME).await?;
|
||||
if confirmed != *canonical {
|
||||
if confirmed != repair_data {
|
||||
return Err(Error::other("pool metadata native heal readback diverged"));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
@@ -28353,7 +28445,7 @@ mod tests {
|
||||
"one version must commit before the next version times out"
|
||||
);
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28387,6 +28479,88 @@ mod tests {
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A target lookup that loses its read-lock quorum must retain the source
|
||||
/// and resume an entry after an earlier version has already committed.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn decommission_defers_target_lookup_read_lock_quorum_and_resumes() {
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let fixture = NullVersionDecommission::new(NullVersionTop::NewerVersion).await;
|
||||
let object = fixture.object;
|
||||
let attempted = Arc::new(AtomicUsize::new(0));
|
||||
let fault = DecommissionTestFaultGuard::install_with_error(
|
||||
Arc::new({
|
||||
let attempted = Arc::clone(&attempted);
|
||||
move |stage, _, faulted, _, success| {
|
||||
stage == "before_target_commit"
|
||||
&& faulted == object
|
||||
&& success
|
||||
&& attempted.fetch_add(1, Ordering::SeqCst) > 0
|
||||
}
|
||||
}),
|
||||
Some(Arc::new(|_, bucket, object| {
|
||||
data_movement::data_movement_stage_error_for_test(
|
||||
"decommission_object",
|
||||
"prepare_put_object",
|
||||
bucket,
|
||||
object,
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
)
|
||||
})),
|
||||
);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
fixture.assert_source_retained().await;
|
||||
assert_eq!(
|
||||
fixture
|
||||
.target_versions()
|
||||
.await
|
||||
.expect("first version committed")
|
||||
.versions
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
fixture.store.pool_meta.read().await.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("active decommission")
|
||||
.items_decommission_failed,
|
||||
0
|
||||
);
|
||||
|
||||
drop(fault);
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(matches!(outcome, DecommissionEntryOutcome::Complete), "{outcome:?}");
|
||||
fixture.assert_moved().await;
|
||||
let mut reader = fixture.store.pools[1]
|
||||
.get_object_reader(
|
||||
fixture.bucket,
|
||||
fixture.object,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
version_id: Some(fixture.seeded.versions[0].version_id.expect("newer version").to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the newer version must remain readable");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read the newer version");
|
||||
assert_eq!(actual, b"written after versioning was enabled");
|
||||
}
|
||||
|
||||
/// A namespace lock still held on the target key, as by a copy whose node
|
||||
/// was killed mid-write, makes target selection fail with a typed
|
||||
/// object-lock timeout. A real held lock, not an injected error, must defer
|
||||
@@ -28429,7 +28603,7 @@ mod tests {
|
||||
|
||||
let (outcome, _) = fixture.run_entry().await;
|
||||
assert!(
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_timeout_deferral(err)),
|
||||
matches!(&outcome, DecommissionEntryOutcome::Deferred(err) if is_decommission_copy_lock_unavailable_deferral(err)),
|
||||
"{outcome:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -28838,17 +29012,17 @@ mod tests {
|
||||
timeout(),
|
||||
);
|
||||
for err in [timeout(), prepare] {
|
||||
assert!(is_decommission_copy_lock_timeout_error(&err), "{err}");
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&err, "bucket-a", "object-a"), "{err}");
|
||||
assert!(!is_decommission_copy_quorum_error(&err), "{err}");
|
||||
let mut error = Some(err);
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("a lock timeout copy must be deferred");
|
||||
assert!(error.is_none());
|
||||
assert!(is_decommission_copy_lock_timeout_deferral(&deferred));
|
||||
assert!(is_decommission_copy_lock_unavailable_deferral(&deferred));
|
||||
assert!(!is_decommission_copy_quorum_deferral(&deferred));
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockTimeout>(&deferred)
|
||||
let failure = data_movement::data_movement_stage_source_as::<DecommissionCopyLockUnavailable>(&deferred)
|
||||
.expect("the deferred error must keep its lock-timeout origin");
|
||||
assert!(is_decommission_copy_lock_timeout_error(&failure.0));
|
||||
assert!(is_decommission_copy_lock_unavailable_error(&failure.0, "bucket-a", "object-a"));
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
@@ -28873,6 +29047,43 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_copy_defers_only_the_matching_object_read_lock_quorum() {
|
||||
let quorum = |mode, bucket: &str, object: &str| Error::NamespaceLockQuorumUnavailable {
|
||||
mode,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
};
|
||||
let wrap = |stage, err| {
|
||||
data_movement::data_movement_stage_error_for_test("decommission_object", stage, "bucket-a", "object-a", err)
|
||||
};
|
||||
let mut error = Some(wrap("prepare_put_object", quorum("read", "bucket-a", "object-a")));
|
||||
let deferred = take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error)
|
||||
.expect("matching read-lock quorum must defer");
|
||||
assert!(error.is_none());
|
||||
assert!(matches!(
|
||||
resolve_decommission_entry_failure(deferred),
|
||||
Ok(DecommissionEntryOutcome::Deferred(_))
|
||||
));
|
||||
for err in [
|
||||
quorum("read", "bucket-a", "object-a"),
|
||||
wrap("source_recheck", quorum("read", "bucket-a", "object-a")),
|
||||
wrap("prepare_put_object", quorum("write", "bucket-a", "object-a")),
|
||||
wrap(
|
||||
"prepare_put_object",
|
||||
quorum("exact_delete_capacity_reconciliation", "bucket-a", "object-a"),
|
||||
),
|
||||
wrap("prepare_put_object", quorum("read", "other-bucket", "object-a")),
|
||||
wrap("prepare_put_object", quorum("read", "bucket-a", "other-object")),
|
||||
] {
|
||||
let mut error = Some(err);
|
||||
assert!(take_decommission_copy_deferral("migrate_object", "bucket-a", "object-a", &mut error).is_none());
|
||||
assert!(error.is_some(), "unrelated lock failures must stay terminal");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
|
||||
let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
|
||||
@@ -29351,6 +29562,53 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_republish_guard_checks_only_three_field_sequence_or_map() {
|
||||
let frame = |payload: Vec<u8>| {
|
||||
let mut data = Vec::new();
|
||||
data.extend_from_slice(&POOL_META_FORMAT.to_le_bytes());
|
||||
data.extend_from_slice(&POOL_META_V1_VERSION.to_le_bytes());
|
||||
data.extend_from_slice(&payload);
|
||||
data
|
||||
};
|
||||
let two_fields = PersistedPoolMetaV1 {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&two_fields), rmp_serde::to_vec_named(&two_fields)] {
|
||||
let data = frame(payload.expect("two-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert!(ensure_legacy_pool_meta_republish_safe(&data).is_ok());
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
for dont_save in [false, true] {
|
||||
let legacy = LegacyPoolMeta {
|
||||
version: POOL_META_V1_VERSION,
|
||||
pools: Vec::new(),
|
||||
dont_save,
|
||||
};
|
||||
for payload in [rmp_serde::to_vec(&legacy), rmp_serde::to_vec_named(&legacy)] {
|
||||
let data = frame(payload.expect("three-field V1 payload must serialize"));
|
||||
assert!(matches!(decode_pool_meta_replica(data.clone()), PoolMetaReplica::Valid { .. }));
|
||||
assert_eq!(ensure_legacy_pool_meta_republish_safe(&data).is_err(), dont_save);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&data).is_ok());
|
||||
let mut trailing = data;
|
||||
trailing.push(0xc0);
|
||||
assert!(ensure_legacy_pool_meta_payload_complete(&trailing).is_err());
|
||||
}
|
||||
}
|
||||
// This guard is only called for already-decoded replicas; malformed
|
||||
// containers must fail selection rather than becoming repair sources.
|
||||
let truncated = frame(vec![0x93, 0x01]);
|
||||
assert!(matches!(
|
||||
decode_pool_meta_replica(truncated),
|
||||
PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_decode_supports_legacy_payload() {
|
||||
let start_time = OffsetDateTime::now_utc();
|
||||
@@ -30571,6 +30829,154 @@ mod pools_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_prewrite_quorum_rejection_allows_retry_after_disks_return() {
|
||||
let (dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
pool.disks.write().await[2..].fill(None);
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let data =
|
||||
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
|
||||
let mut transaction = state.arm_transaction();
|
||||
save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect_err("two of four disks cannot satisfy the identity write quorum");
|
||||
drop(transaction);
|
||||
for dir in &dirs {
|
||||
assert!(
|
||||
!dir.path()
|
||||
.join(crate::disk::RUSTFS_META_BUCKET)
|
||||
.join(POOL_META_IDENTITY_NAME)
|
||||
.exists(),
|
||||
"the rejected write must not create an identity replica"
|
||||
);
|
||||
}
|
||||
state
|
||||
.ensure_write_safe("retry after disks return")
|
||||
.expect("a rejected inline write must not poison the startup transaction");
|
||||
|
||||
let ordinary_error = crate::config::com::save_config_with_opts(
|
||||
pool.clone(),
|
||||
"ordinary-config",
|
||||
data.clone(),
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("ordinary writes must still fail without quorum");
|
||||
assert!(
|
||||
matches!(ordinary_error, Error::InsufficientWriteQuorum(_, _)),
|
||||
"ordinary callers must retain their existing quorum error"
|
||||
);
|
||||
|
||||
*pool.disks.write().await = disks.into_iter().map(Some).collect();
|
||||
let mut retry = state.arm_transaction();
|
||||
save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut retry,
|
||||
)
|
||||
.await
|
||||
.expect("the same transaction must succeed when the disks return");
|
||||
retry.disarm();
|
||||
drop(retry);
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
|
||||
.await
|
||||
.expect("read back the committed identity"),
|
||||
data
|
||||
);
|
||||
state
|
||||
.ensure_write_safe("after successful retry")
|
||||
.expect("retry must leave writes enabled");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_prewrite_quorum_rejection_keeps_an_earlier_write_armed() {
|
||||
let (_dirs, disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let data =
|
||||
super::initialized_pool_meta_identity_for_test(uuid::Uuid::new_v4(), 1).expect("encode the deployment identity");
|
||||
let mut transaction = state.arm_transaction();
|
||||
let written = save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"prepare_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect("the earlier phase must really commit");
|
||||
pool.disks.write().await[2..].fill(None);
|
||||
save_pool_meta_object_cas(
|
||||
pool.clone(),
|
||||
POOL_META_IDENTITY_NAME,
|
||||
data.clone(),
|
||||
&PoolMetaCasToken::Existing(written.etag.expect("committed identity has an ETag")),
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect_err("the next phase must fail without a write quorum");
|
||||
drop(transaction);
|
||||
*pool.disks.write().await = disks.into_iter().map(Some).collect();
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(pool, POOL_META_IDENTITY_NAME)
|
||||
.await
|
||||
.expect("the earlier commit must survive the rejected write"),
|
||||
data
|
||||
);
|
||||
let error = state
|
||||
.ensure_write_safe("after a committed phase")
|
||||
.expect_err("a later rejection cannot discharge the earlier write");
|
||||
assert_eq!(
|
||||
error.pool_metadata_failure().expect("retain typed uncertainty").kind,
|
||||
crate::error::PoolMetadataFailure::TransactionUnknown
|
||||
);
|
||||
assert_eq!(error.pool_metadata_failure().expect("retain the earlier mutation").phase, "prepare_cas");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_meta_prewrite_noninline_failure_keeps_transaction_armed() {
|
||||
let (_dirs, _disks, pool) = crate::set_disk::hermetic_set_disks_isolated(4).await;
|
||||
pool.disks.write().await[2..].fill(None);
|
||||
let state = super::PoolMetaWriteState::default();
|
||||
let mut transaction = state.arm_transaction();
|
||||
save_pool_meta_object_cas(
|
||||
pool,
|
||||
POOL_META_IDENTITY_NAME,
|
||||
vec![0x51; 2 * 1024 * 1024],
|
||||
&PoolMetaCasToken::Missing,
|
||||
&PoolMetaPersistenceFence::OfflineRecovery,
|
||||
"identity_cas",
|
||||
&mut transaction,
|
||||
)
|
||||
.await
|
||||
.expect_err("the noninline write must fail without quorum");
|
||||
drop(transaction);
|
||||
assert!(
|
||||
state.ensure_write_safe("after writer setup").is_err(),
|
||||
"a failure after disk writers were created must remain fail-closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum IdentityRepairFault {
|
||||
Healthy,
|
||||
|
||||
@@ -317,8 +317,9 @@ impl Sets {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn stop_endpoint_monitor_for_test(&self) {
|
||||
/// Stop endpoint reconnection and wait for any in-flight pass before a test owns disk probes.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn stop_endpoint_monitor_for_test(&self) {
|
||||
if let Some(exit_signal) = &self.exit_signal {
|
||||
let _ = exit_signal.send(());
|
||||
// Wait for an in-flight reconnect pass as well as its receiver to exit.
|
||||
@@ -655,19 +656,6 @@ impl Sets {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_prepared_reservations(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
reservations: Option<Vec<Option<crate::set_disk::RenameMetadataReservation>>>,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
self.get_disks_by_key(object)
|
||||
.put_object_with_prepared_reservations(bucket, object, data, opts, reservations)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_with_old_current_size_for_data_movement(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -31,6 +31,8 @@ use tokio::fs;
|
||||
use tokio::time::Instant;
|
||||
|
||||
pub(crate) const BARRIER_DIR_ENV: &str = "RUSTFS_E2E_DATA_MOVEMENT_BARRIER_DIR";
|
||||
/// Rebalance holds its admission fence; no version of this entry has moved.
|
||||
pub(crate) const STAGE_AFTER_ENTRY_ADMISSION: &str = "after_entry_admission";
|
||||
/// Every target part is staged and the target version is not yet published.
|
||||
pub(crate) const STAGE_BEFORE_TARGET_COMMIT: &str = "before_target_commit";
|
||||
/// The target commit is durable and the source preflight matched; the source
|
||||
|
||||
@@ -605,6 +605,7 @@ fn resolve_data_movement_abort_result(
|
||||
#[derive(Debug)]
|
||||
struct DataMovementStageError {
|
||||
rendered: String,
|
||||
stage: Option<String>,
|
||||
source: Box<dyn std::error::Error + Send + Sync>,
|
||||
}
|
||||
|
||||
@@ -626,6 +627,7 @@ where
|
||||
{
|
||||
Error::other(DataMovementStageError {
|
||||
rendered,
|
||||
stage: None,
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
@@ -634,7 +636,21 @@ fn data_movement_stage_error<E>(op_label: &str, stage: &str, bucket: &str, objec
|
||||
where
|
||||
E: std::error::Error + Send + Sync + 'static,
|
||||
{
|
||||
data_movement_context_error(format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"), err)
|
||||
Error::other(DataMovementStageError {
|
||||
rendered: format!("{op_label}: {stage} failed for {bucket}/{object}: {err}"),
|
||||
stage: Some(stage.to_owned()),
|
||||
source: Box::new(err),
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn data_movement_error_is_stage(err: &Error, stage: &str) -> bool {
|
||||
let Error::Io(io_err) = err else {
|
||||
return false;
|
||||
};
|
||||
io_err
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<DataMovementStageError>())
|
||||
.is_some_and(|source| source.stage.as_deref() == Some(stage))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -55,6 +55,20 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Join the local disk's actual bootstrap worker for deterministic coverage tests.
|
||||
/// Completion may mean legacy fallback after a failed scan; callers must assert
|
||||
/// the expected index result separately. Remote disks have no local worker.
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub async fn wait_for_multipart_index_bootstrap(disk: &crate::disk::DiskStore) -> Result<()> {
|
||||
match disk.as_ref() {
|
||||
crate::disk::Disk::Local(local) => {
|
||||
local.disk.wait_for_multipart_index_bootstrap().await;
|
||||
Ok(())
|
||||
}
|
||||
crate::disk::Disk::Remote(_) => Err(DiskError::MethodNotAllowed),
|
||||
}
|
||||
}
|
||||
|
||||
/// Disk health status constants
|
||||
const DISK_HEALTH_OK: u32 = 0;
|
||||
const DISK_HEALTH_FAULTY: u32 = 1;
|
||||
@@ -604,69 +618,8 @@ impl LocalDiskWrapper {
|
||||
let super::RenameDataGuards {
|
||||
external_guard,
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
metadata_reservation_token,
|
||||
..
|
||||
} = guards;
|
||||
let metadata_reservation = match (metadata_reservation, metadata_reservation_token) {
|
||||
(Some(_), Some(_)) => return super::RenameDataObservation::unknown(Err(DiskError::InvalidPath)),
|
||||
(Some(reservation), None) => Some(reservation),
|
||||
(None, Some(token)) => match self.disk.claim_object_metadata_reservation(token, dst_volume, dst_path) {
|
||||
Ok(reservation) => Some(reservation),
|
||||
// A missing token may already have been claimed by an earlier
|
||||
// request that is still publishing; do not claim it was safe.
|
||||
Err(DiskError::DiskOngoingReq) => {
|
||||
return super::RenameDataObservation::unknown(Err(DiskError::DiskOngoingReq));
|
||||
}
|
||||
Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
},
|
||||
(None, None) => None,
|
||||
};
|
||||
// A queued reservation must acquire its metadata lease before disk
|
||||
// admission. Otherwise a waiting background mutation could hold the
|
||||
// disk permit needed by the earlier lease owner.
|
||||
let metadata_reservation = match metadata_reservation {
|
||||
Some(reservation) => Some(reservation.wait().await),
|
||||
None => None,
|
||||
};
|
||||
// The claimed reservation stays in this cancellable future while disk
|
||||
// admission can wait. Only physical work moves into an owned task.
|
||||
let prechecked_permit = if metadata_reservation.is_some() {
|
||||
// Background admission is exclusive. Waiting for it while holding
|
||||
// this object's lease can deadlock an admitted mutation waiting for
|
||||
// the same lease. An already owned permit does not need to wait.
|
||||
if current_disk_io_context().is_some_and(|context| {
|
||||
context.class == DiskIoClass::Background
|
||||
&& !context
|
||||
.permit
|
||||
.as_ref()
|
||||
.is_some_and(|permit| permit.is_for(&self.io_admission))
|
||||
}) {
|
||||
return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let preflight = self
|
||||
.with_native_resources(async {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.acquire_disk_io_permit().await
|
||||
})
|
||||
.await;
|
||||
match preflight {
|
||||
Ok(Ok(Some(permit))) => Some(permit),
|
||||
Ok(Ok(None)) => return super::RenameDataObservation::not_dispatched(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) | Err(error) => return super::RenameDataObservation::not_dispatched(error),
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// The reserved metadata lease must outlive a cancelled RPC waiter.
|
||||
let external_guard: Option<Arc<dyn Send + Sync>> = if metadata_reservation.is_some() {
|
||||
Some(Arc::new(external_guard))
|
||||
} else {
|
||||
external_guard
|
||||
};
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
@@ -685,54 +638,27 @@ impl LocalDiskWrapper {
|
||||
};
|
||||
let observed = run_owned_mutation(external_guard, move || async move {
|
||||
let mut preflight_rejection = None;
|
||||
let rename = || async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = if let Some(reservation) = metadata_reservation {
|
||||
Box::pin(operation.disk.rename_data_observed_with_reservation(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
(namespace_owner, Some(reservation)),
|
||||
))
|
||||
.await
|
||||
} else {
|
||||
Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await
|
||||
};
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
};
|
||||
let result = if let Some(permit) = prechecked_permit {
|
||||
with_disk_io_permit(Some(permit), async {
|
||||
operation
|
||||
.with_native_resources(operation.track_admitted_disk_operation(
|
||||
"rename_data",
|
||||
move |io_permit| async move {
|
||||
let _io_permit = io_permit;
|
||||
rename().await
|
||||
},
|
||||
Duration::ZERO,
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::Write,
|
||||
true,
|
||||
let result = operation
|
||||
.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
// Preserve the former DiskAPI future's single boxing boundary.
|
||||
let observed = Box::pin(operation.disk.rename_data_observed(
|
||||
&src_volume,
|
||||
&src_path,
|
||||
&fi,
|
||||
&dst_volume,
|
||||
&dst_path,
|
||||
namespace_owner,
|
||||
))
|
||||
.await?
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
operation
|
||||
.track_disk_health_mutation("rename_data", DiskMetricMutation::Write, rename, timeout_duration)
|
||||
.await
|
||||
};
|
||||
.await;
|
||||
preflight_rejection = observed.preflight_rejection;
|
||||
observed.result
|
||||
},
|
||||
timeout_duration,
|
||||
)
|
||||
.await;
|
||||
// Health tracking must observe the real disk error, not an Ok tuple.
|
||||
Ok(super::RenameDataObservation {
|
||||
result,
|
||||
@@ -941,6 +867,8 @@ pub(crate) struct DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock<HashMap<&'static str, Arc<DiskOperationMetrics>>>,
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex<Option<std::sync::Weak<ScopedMultipartCleanupCompletions>>>,
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex<Option<Duration>>,
|
||||
/// Caller API operations currently executing through the disk health wrapper.
|
||||
api_waiting: AtomicU32,
|
||||
/// Operations rejected because the disk was unavailable for the caller.
|
||||
@@ -968,6 +896,8 @@ impl Default for DiskHealthMetricEpoch {
|
||||
fallback_operation_metrics: StdRwLock::new(HashMap::new()),
|
||||
#[cfg(test)]
|
||||
multipart_cleanup_observer: std::sync::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
operation_timeout_override: std::sync::Mutex::new(None),
|
||||
api_waiting: AtomicU32::new(0),
|
||||
total_errors_availability: AtomicU64::new(0),
|
||||
total_errors_timeout: AtomicU64::new(0),
|
||||
@@ -1661,6 +1591,20 @@ impl Default for DiskHealthTracker {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ScopedDiskOperationTimeout {
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
previous: Option<Duration>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for ScopedDiskOperationTimeout {
|
||||
fn drop(&mut self) {
|
||||
*self.metrics.operation_timeout_override.lock().expect("test timeout lock") = self.previous;
|
||||
}
|
||||
}
|
||||
|
||||
/// LocalDiskWrapper wraps a DiskStore with health tracking capabilities.
|
||||
/// This is similar to Go's xlStorageDiskIDCheck.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -1686,58 +1630,6 @@ pub struct LocalDiskWrapper {
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
/// Reserve the metadata mutation lock for one object on this disk instance.
|
||||
/// Unclaimed reservations expire after 60 seconds.
|
||||
pub async fn reserve_object_metadata(&self, bucket: &str, object: &str, timeout: Duration) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.reserve_object_metadata(bucket, object, timeout).await
|
||||
}
|
||||
|
||||
/// Register an object metadata reservation in FIFO order on this disk.
|
||||
/// The returned token may be claimed before its physical lease is available.
|
||||
pub async fn enqueue_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.enqueue_object_metadata(bucket, object).await
|
||||
}
|
||||
|
||||
/// Prepare an inert reservation token bound to this disk and object.
|
||||
pub async fn prepare_object_metadata(&self, bucket: &str, object: &str) -> Result<Uuid> {
|
||||
self.prepare_object_metadata_with_owner(bucket, object, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_object_metadata_with_owner(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
owner: Option<Arc<dyn super::os::MetadataReservationFence>>,
|
||||
) -> Result<Uuid> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.prepare_object_metadata_with_owner(bucket, object, owner)
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token; a cancelled token cannot be reactivated.
|
||||
pub async fn activate_object_metadata(&self, bucket: &str, object: &str, token: Uuid) -> Result<()> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.activate_object_metadata(bucket, object, token).await
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. A claimed physical operation retains its own lock.
|
||||
pub fn cancel_object_metadata_reservation(&self, token: Uuid) -> bool {
|
||||
self.disk.cancel_object_metadata_reservation(token)
|
||||
}
|
||||
|
||||
/// Create a new LocalDiskWrapper
|
||||
pub fn new(disk: Arc<LocalDisk>, health_check: bool) -> Self {
|
||||
let admission = DiskIoAdmission::new(disk.endpoint().to_string());
|
||||
@@ -2054,6 +1946,23 @@ impl LocalDiskWrapper {
|
||||
self.health_check
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
|
||||
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
|
||||
// The metrics epoch follows this drive across clones and reconnects.
|
||||
// Its scoped override cannot shorten unrelated healthy drives' I/O.
|
||||
let previous = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.replace(timeout);
|
||||
ScopedDiskOperationTimeout {
|
||||
metrics: Arc::clone(&self.metrics),
|
||||
previous,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable health monitoring after disk creation.
|
||||
/// Used to defer health checks until after startup format loading completes.
|
||||
pub fn enable_health_check(&self) {
|
||||
@@ -2538,12 +2447,17 @@ impl LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
self.disk.has_clean_active_multipart_source(upload_or_part_path, part_path)
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2551,7 +2465,7 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
.try_publish_prepared_delete_local_with_budget(volume, object, transaction, namespace_owner, work_budget)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2567,6 +2481,7 @@ impl LocalDiskWrapper {
|
||||
transaction: uuid::Uuid,
|
||||
journal: &crate::disk::DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
@@ -2574,7 +2489,14 @@ impl LocalDiskWrapper {
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.try_finish_and_finalize_prepared_delete_local_with_budget(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
})
|
||||
},
|
||||
@@ -2717,7 +2639,6 @@ impl LocalDiskWrapper {
|
||||
timeout_duration,
|
||||
timeout_health_action,
|
||||
mutation,
|
||||
false,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
@@ -2729,7 +2650,6 @@ impl LocalDiskWrapper {
|
||||
timeout_duration: Duration,
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
identity_prechecked: bool,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
@@ -2751,7 +2671,7 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
|
||||
// Check if disk is stale
|
||||
if !identity_prechecked && let Err(err) = self.check_disk_stale().await {
|
||||
if let Err(err) = self.check_disk_stale().await {
|
||||
self.metrics.record_availability_error();
|
||||
return Err(err);
|
||||
}
|
||||
@@ -2776,6 +2696,13 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
#[cfg(test)]
|
||||
let timeout_duration = self
|
||||
.metrics
|
||||
.operation_timeout_override
|
||||
.lock()
|
||||
.expect("test timeout lock")
|
||||
.map_or(timeout_duration, |injected| timeout_duration.min(injected));
|
||||
// Execute the operation with timeout
|
||||
let result = tokio::time::timeout(timeout_duration, operation).await;
|
||||
|
||||
@@ -2824,6 +2751,18 @@ impl LocalDiskWrapper {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDiskWrapper {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<crate::disk::multipart_index::MultipartIndexPage> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
self.check_disk_stale().await?;
|
||||
self.disk.list_multipart_index_roots(bucket, marker).await
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
// The local implementation shares this binary's capability. Availability
|
||||
// and disk identity remain mutation checks and write-quorum inputs.
|
||||
@@ -2877,6 +2816,62 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
)
|
||||
.await
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryDiscovery> {
|
||||
self.track_disk_health_with_op(
|
||||
"discover_delete_transactions",
|
||||
|| async {
|
||||
self.disk
|
||||
.discover_delete_transactions_bounded(volume, object, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoveryInspection> {
|
||||
self.track_disk_health_with_op(
|
||||
"inspect_delete_transaction",
|
||||
|| async {
|
||||
self.disk
|
||||
.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: crate::disk::BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<crate::disk::BoundedDeleteRecoverySettlementReply> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async {
|
||||
self.disk
|
||||
.settle_delete_transaction_bounded(volume, request, namespace_owner)
|
||||
.await
|
||||
},
|
||||
crate::disk::get_delete_recovery_timeout()?,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -3845,7 +3840,11 @@ mod tests {
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
time::advance(SLOW_DISK_OPERATION_THRESHOLD - Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
assert!(logs.contents().is_empty(), "normal disk latency should not warn");
|
||||
assert!(
|
||||
!logs.contents().contains("disk_operation_slow") && !logs.contents().contains("WARN"),
|
||||
"normal disk latency should not warn: {}",
|
||||
logs.contents()
|
||||
);
|
||||
time::advance(Duration::from_millis(1)).await;
|
||||
assert!(futures::poll!(&mut operation).is_pending());
|
||||
let warning = logs.contents();
|
||||
@@ -5719,115 +5718,6 @@ mod tests {
|
||||
drop(metadata_lease);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_reserved_rename_releases_lease_before_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let meta = dir.path().join(RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
|
||||
.await
|
||||
.expect("write format");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
|
||||
disk.make_volume("bucket").await.expect("bucket volume");
|
||||
let wrapper = Arc::new(LocalDiskWrapper::new(Arc::clone(&disk), false));
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
wrapper.set_disk_id_state(Some(format.erasure.this)).await;
|
||||
disk.format_info.write().await.last_check = None;
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
disk.install_format_probe_hook(false, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv_timeout(Duration::from_secs(30));
|
||||
});
|
||||
let publisher = Arc::clone(&wrapper);
|
||||
let rename = tokio::spawn(async move {
|
||||
publisher
|
||||
.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&FileInfo::default(),
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), entered_rx)
|
||||
.await
|
||||
.expect("disk identity preflight starts")
|
||||
.expect("preflight hook");
|
||||
rename.abort();
|
||||
assert!(matches!(rename.await, Err(error) if error.is_cancelled()));
|
||||
let key = disk.get_object_path("bucket", "object").expect("object path");
|
||||
let mut successor = Box::pin(crate::disk::os::acquire_metadata_mutation_lease(&key, None));
|
||||
let released_before_io = tokio::time::timeout(Duration::from_secs(1), &mut successor).await.is_ok();
|
||||
release_tx.send(()).expect("release disk identity preflight");
|
||||
if !released_before_io {
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), successor)
|
||||
.await
|
||||
.expect("preflight finishes"),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
released_before_io,
|
||||
"a cancelled request must not retain the reservation before physical mutation starts"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reserved_background_rename_does_not_wait_for_disk_admission() {
|
||||
use crate::disk::{RUSTFS_META_TMP_BUCKET, RenameDataGuards};
|
||||
|
||||
let dir = tempfile::tempdir().expect("disk directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let token = wrapper
|
||||
.reserve_object_metadata("bucket", "object", Duration::from_secs(5))
|
||||
.await
|
||||
.expect("reserve key");
|
||||
let held = wrapper
|
||||
.io_admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("background permit");
|
||||
let fi = FileInfo::default();
|
||||
let rename = crate::disk::with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
wrapper.rename_data_observed_with_guards(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"staging",
|
||||
&fi,
|
||||
"bucket",
|
||||
"object",
|
||||
RenameDataGuards {
|
||||
metadata_reservation_token: Some(token),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
);
|
||||
let result = tokio::time::timeout(Duration::from_secs(1), rename).await;
|
||||
drop(held);
|
||||
let observation = result.expect("reserved mutation must not wait while holding the object lease");
|
||||
assert!(matches!(&observation.result, Err(DiskError::DiskOngoingReq)));
|
||||
assert!(observation.rejected_before_publication());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_versions_counts_faulty_drive_availability_rejection() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
@@ -6291,6 +6181,67 @@ mod tests {
|
||||
assert!(health.offline_duration().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_operation_timeout_is_per_disk_and_preserves_caller_policy() {
|
||||
let directories = [
|
||||
tempfile::tempdir().expect("faulted drive"),
|
||||
tempfile::tempdir().expect("healthy drive"),
|
||||
];
|
||||
let mut wrappers = Vec::new();
|
||||
for directory in &directories {
|
||||
let endpoint = Endpoint::try_from(directory.path().to_str().expect("UTF-8 drive path")).expect("drive endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local drive"));
|
||||
wrappers.push(LocalDiskWrapper::new(disk, false));
|
||||
}
|
||||
let reconnected = LocalDiskWrapper::new_with_reconnect_state(
|
||||
Arc::clone(&wrappers[0].disk),
|
||||
false,
|
||||
Some(wrappers[0].health_tracker_epoch_for_reconnect()),
|
||||
);
|
||||
let scope = wrappers[0].override_operation_timeout_for_test(Duration::from_secs(1));
|
||||
tokio::time::pause();
|
||||
let operation = || async {
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
Ok(())
|
||||
};
|
||||
let (faulted, healthy) = tokio::join!(
|
||||
reconnected.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
wrappers[1].track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_secs(3),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
),
|
||||
);
|
||||
assert_eq!(faulted.expect_err("only the selected drive must time out"), DiskError::Timeout);
|
||||
healthy.expect("healthy drive keeps its caller's deadline");
|
||||
let shorter_started = tokio::time::Instant::now();
|
||||
let shorter = wrappers[0]
|
||||
.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
operation,
|
||||
Duration::from_millis(500),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(shorter.expect_err("fault injection cannot extend a caller deadline"), DiskError::Timeout);
|
||||
assert!(shorter_started.elapsed() < Duration::from_secs(1));
|
||||
wrappers[0]
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::ZERO)
|
||||
.await
|
||||
.expect("explicit unbounded physical ownership cannot acquire a test deadline");
|
||||
drop(scope);
|
||||
reconnected
|
||||
.track_disk_health_with_op("read_metadata", operation, Duration::from_secs(3))
|
||||
.await
|
||||
.expect("dropping the scope restores reconnected callers' deadlines");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ignored_timeout_does_not_mark_drive_failure() {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
|
||||
@@ -172,6 +172,8 @@ pub enum DiskError {
|
||||
|
||||
#[error("guarded batch deletion is not supported by this disk")]
|
||||
GuardedDeleteUnsupported,
|
||||
#[error("foreground delete recovery work limit reached")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
|
||||
#[error("erasure write quorum")]
|
||||
ErasureWriteQuorum,
|
||||
@@ -770,6 +772,7 @@ impl Clone for DiskError {
|
||||
DiskError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
DiskError::ConditionalDeleteConflict => DiskError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => DiskError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
DiskError::ErasureWriteQuorum => DiskError::ErasureWriteQuorum,
|
||||
DiskError::ErasureReadQuorum => DiskError::ErasureReadQuorum,
|
||||
DiskError::ShortWrite => DiskError::ShortWrite,
|
||||
@@ -829,6 +832,7 @@ impl DiskError {
|
||||
DiskError::RemoteClientUnavailable(_) => 0x2B,
|
||||
DiskError::ConditionalDeleteConflict => 0x2C,
|
||||
DiskError::GuardedDeleteUnsupported => 0x2D,
|
||||
DiskError::DeleteRecoveryWorkLimit => 0x2E,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -879,6 +883,7 @@ impl DiskError {
|
||||
0x2B => Some(DiskError::RemoteClientUnavailable(String::new())),
|
||||
0x2C => Some(DiskError::ConditionalDeleteConflict),
|
||||
0x2D => Some(DiskError::GuardedDeleteUnsupported),
|
||||
0x2E => Some(DiskError::DeleteRecoveryWorkLimit),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -1662,6 +1667,7 @@ mod tests {
|
||||
for (error, code) in [
|
||||
(DiskError::ConditionalDeleteConflict, 0x2C),
|
||||
(DiskError::GuardedDeleteUnsupported, 0x2D),
|
||||
(DiskError::DeleteRecoveryWorkLimit, 0x2E),
|
||||
] {
|
||||
assert_eq!(error.to_u32(), code);
|
||||
assert_eq!(DiskError::from_u32(code), Some(error.clone()));
|
||||
|
||||
@@ -5,6 +5,7 @@ use super::{FdCache, FdCacheEntry, FdKey};
|
||||
use std::fs::File;
|
||||
use std::os::unix::fs::FileExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
fn key(volume: &str, path: &str, direct: bool) -> FdKey {
|
||||
FdKey {
|
||||
@@ -52,7 +53,11 @@ async fn uring_fd_cache_exact_invalidation_removes_both_modes_and_preserves_othe
|
||||
async fn uring_fd_cache_exact_invalidation_bumps_one_generation_and_rejects_old_epoch_inserts() {
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::new();
|
||||
let entry = Arc::new(FdCacheEntry { file, len: 3 });
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file,
|
||||
len: 3,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let old_generation = cache.generation();
|
||||
cache.invalidate_exact("v", "part").await;
|
||||
let current_generation = cache.generation();
|
||||
@@ -125,7 +130,7 @@ async fn uring_fd_cache_volume_and_clear_invalidation_cover_both_modes() {
|
||||
cache.insert(key("v", "part", direct), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
cache.get(&key("v", "part", direct)).await.is_some(),
|
||||
"new entries must survive the earlier volume predicate"
|
||||
"new entries must survive the earlier volume invalidation"
|
||||
);
|
||||
}
|
||||
cache.clear();
|
||||
@@ -164,6 +169,7 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
let replacement = Arc::new(FdCacheEntry {
|
||||
file: Arc::new(File::open(directory.path().join("part")).expect("open replacement inode")),
|
||||
len: 11,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
for direct in [false, true] {
|
||||
let key = key("v", "part", direct);
|
||||
@@ -177,3 +183,459 @@ async fn uring_fd_cache_invalidation_keeps_borrowed_old_inode_alive_and_allows_r
|
||||
assert_eq!(fresh.len, 11);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_tracker_skips_unmatched_invalidations_and_handles_deep_prefixes() {
|
||||
let (_directory, file) = fixture();
|
||||
let mut cache = FdCache::new();
|
||||
|
||||
// Unmatched invalidations on an empty cache must not spawn the maintenance task.
|
||||
cache.invalidate_under("v", "missing/prefix");
|
||||
cache.invalidate_volume("v");
|
||||
cache.invalidate_exact("v", "missing/prefix/part.1").await;
|
||||
cache.clear();
|
||||
assert!(
|
||||
cache.maintenance_task.get().is_none(),
|
||||
"empty-cache invalidations must not spawn maintenance"
|
||||
);
|
||||
|
||||
// Populate a normal entry and verify matched vs unmatched filter gates directly,
|
||||
// and ensure unmatched invalidations do not enqueue pending removals.
|
||||
let cached_key = key("v", "obj-1/0198a3f0/part.1", false);
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(cache.tracker.live_entries(), 1);
|
||||
assert!(cache.tracker.may_match_volume("v"));
|
||||
assert!(!cache.tracker.may_match_volume("other-volume"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1"));
|
||||
assert!(cache.tracker.may_match_prefix("v", "obj-1/0198a3f0"));
|
||||
assert!(!cache.tracker.may_match_prefix("v", "obj-2/0198a3f0"));
|
||||
assert!(cache.tracker.may_match_exact("v", "obj-1/0198a3f0/part.1"));
|
||||
assert!(!cache.tracker.may_match_exact("v", "obj-1/0198a3f0/xl.meta"));
|
||||
|
||||
cache.invalidate_volume("other-volume");
|
||||
cache.invalidate_under("v", "obj-2/0198a3f0");
|
||||
cache.invalidate_exact("v", "obj-1/0198a3f0/xl.meta").await;
|
||||
assert!(
|
||||
!cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"unmatched invalidations must not enqueue maintenance removals"
|
||||
);
|
||||
assert!(cache.get(&cached_key).await.is_some());
|
||||
|
||||
// Deeply nested path (> FD_CACHE_MAX_INDEXED_PREFIXES slashes) must fall back
|
||||
// to per-volume unindexed_counts without degrading other volumes.
|
||||
let other_vol_key = key("v2", "shallow/part.1", false);
|
||||
cache.insert(other_vol_key.clone(), Arc::clone(&file)).await;
|
||||
let deep_path = "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13/c14/part.1";
|
||||
let deep_key = key("v", deep_path, false);
|
||||
cache.insert(deep_key.clone(), Arc::clone(&file)).await;
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
!cache.tracker.may_match_prefix("v2", "unmatched/prefix"),
|
||||
"deep key in volume v must not disable prefix filtering in volume v2"
|
||||
);
|
||||
cache.invalidate_under("v", "c1/c2/c3/c4/c5/c6/c7/c8/c9/c10/c11/c12/c13");
|
||||
assert!(cache.get(&deep_key).await.is_none());
|
||||
cache.invalidate_volume("v2");
|
||||
assert!(cache.get(&other_vol_key).await.is_none());
|
||||
|
||||
// Replacing an invalidated key before maintenance drains pending_invalidations
|
||||
// must release the old descriptor immediately and keep the replacement live
|
||||
// even when a strong reference to the old CachedFdEntry shell survives until
|
||||
// maintenance runs its Arc::ptr_eq check.
|
||||
let old_file = Arc::new(tempfile::tempfile().expect("create replaced descriptor"));
|
||||
let old_weak = Arc::downgrade(&old_file);
|
||||
cache.insert(cached_key.clone(), old_file).await;
|
||||
let held_old_shell = cache.cache.get(&cached_key).expect("cached shell must exist");
|
||||
cache.invalidate_under("v", "obj-1/0198a3f0");
|
||||
assert!(cache.get(&cached_key).await.is_none());
|
||||
cache.insert(cached_key.clone(), Arc::clone(&file)).await;
|
||||
assert!(
|
||||
old_weak.upgrade().is_none(),
|
||||
"replacing an entry must retire and drop the previous descriptor immediately"
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache
|
||||
.tracker
|
||||
.has_pending_invalidations
|
||||
.load(std::sync::atomic::Ordering::SeqCst)
|
||||
{
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("maintenance should drain pending invalidations");
|
||||
drop(held_old_shell);
|
||||
assert!(
|
||||
cache.get(&cached_key).await.is_some(),
|
||||
"stale pending invalidation must not evict a newer replacement"
|
||||
);
|
||||
assert_eq!(
|
||||
cache
|
||||
.tracker
|
||||
.unindexed_counts
|
||||
.iter()
|
||||
.map(|c| c.load(std::sync::atomic::Ordering::Acquire))
|
||||
.sum::<u32>(),
|
||||
0
|
||||
);
|
||||
|
||||
// Dropping a cache while a live non-invalidated entry is present (even if a
|
||||
// cloned Moka handle outlives FdCache and the maintenance task is aborted)
|
||||
// must retire the descriptor and release FdCacheTracker without cycles.
|
||||
let live_drop_file = Arc::new(tempfile::tempfile().expect("create live drop descriptor"));
|
||||
let live_drop_weak = Arc::downgrade(&live_drop_file);
|
||||
let shutdown_file = Arc::new(tempfile::tempfile().expect("create shutdown descriptor"));
|
||||
let shutdown_weak = Arc::downgrade(&shutdown_file);
|
||||
let tracker_weak = Arc::downgrade(&cache.tracker);
|
||||
let cloned_moka = cache.cache.clone();
|
||||
cache.insert(key("v", "live-on-drop/part.1", false), live_drop_file).await;
|
||||
cache.insert(key("v", "shutdown/part.1", false), shutdown_file).await;
|
||||
cache.invalidate_under("v", "shutdown");
|
||||
if let Some(handle) = cache.maintenance_task.take() {
|
||||
handle.abort();
|
||||
let _ = handle.await;
|
||||
}
|
||||
drop(cache);
|
||||
assert!(
|
||||
live_drop_weak.upgrade().is_none(),
|
||||
"Drop for FdCache must retire live descriptors even if a cloned Moka handle survives"
|
||||
);
|
||||
assert!(
|
||||
shutdown_weak.upgrade().is_none(),
|
||||
"dropping cache when maintenance is aborted must still release descriptors"
|
||||
);
|
||||
assert!(
|
||||
tracker_weak.upgrade().is_none(),
|
||||
"dropping cache must release FdCacheTracker without reference cycles"
|
||||
);
|
||||
drop(cloned_moka);
|
||||
|
||||
// Short-TTL expired entries (including mixed expired and fresh entries) must
|
||||
// be retired by invalidate_under and clear() even though Moka's iterator skips
|
||||
// expired entries.
|
||||
let ttl_cache = FdCache::with_ttl(std::time::Duration::from_millis(15));
|
||||
let expired_file = Arc::new(tempfile::tempfile().expect("create short-ttl descriptor"));
|
||||
let expired_weak = Arc::downgrade(&expired_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.1", false), expired_file).await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let fresh_file = Arc::new(tempfile::tempfile().expect("create fresh descriptor"));
|
||||
let fresh_weak = Arc::downgrade(&fresh_file);
|
||||
ttl_cache.insert(key("v", "ttl-obj/part.2", false), fresh_file).await;
|
||||
ttl_cache.invalidate_under("v", "ttl-obj");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while expired_weak.upgrade().is_some() || fresh_weak.upgrade().is_some() {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("invalidate_under must retire both expired and fresh matching descriptors");
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
|
||||
let clear_expired_file = Arc::new(tempfile::tempfile().expect("create clear short-ttl descriptor"));
|
||||
let clear_expired_weak = Arc::downgrade(&clear_expired_file);
|
||||
ttl_cache
|
||||
.insert(key("v", "ttl-clear/part.1", false), clear_expired_file)
|
||||
.await;
|
||||
tokio::time::sleep(std::time::Duration::from_millis(30)).await;
|
||||
let clear_live_file = Arc::new(tempfile::tempfile().expect("create clear live descriptor"));
|
||||
let clear_live_weak = Arc::downgrade(&clear_live_file);
|
||||
ttl_cache.insert(key("v", "ttl-clear/part.2", false), clear_live_file).await;
|
||||
ttl_cache.clear();
|
||||
assert!(
|
||||
clear_expired_weak.upgrade().is_none() && clear_live_weak.upgrade().is_none(),
|
||||
"clear() must synchronously retire both TTL-expired and live descriptors"
|
||||
);
|
||||
assert_eq!(ttl_cache.tracker.live_entries(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fd_cache_lockstep_oracle_equivalence_and_counter_conservation() {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = FdCache::with_ttl(std::time::Duration::from_secs(3600));
|
||||
let mut oracle: HashMap<FdKey, Arc<FdCacheEntry>> = HashMap::new();
|
||||
|
||||
let volumes = ["vol-a", "vol-b", "vol-c", ".rustfs.sys/multipart"];
|
||||
let paths = [
|
||||
"a/b",
|
||||
"a/b/part.1",
|
||||
"a/b/dir/part.2",
|
||||
"a/bc",
|
||||
"a/bc/part.1",
|
||||
"prefix/obj-0001.bin/0198a3f0/part.1",
|
||||
"prefix/obj-0002.bin/0198a3f0/part.1",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/part.1",
|
||||
];
|
||||
let prefixes = [
|
||||
"a",
|
||||
"a/",
|
||||
"a/b",
|
||||
"a/b/",
|
||||
"a/b/dir",
|
||||
"a/bc",
|
||||
"prefix",
|
||||
"prefix/obj-0001.bin",
|
||||
"prefix/obj-0001.bin/0198a3f0",
|
||||
"d1/d2/d3/d4/d5/c_missing",
|
||||
"d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13",
|
||||
"nonexistent/prefix",
|
||||
];
|
||||
|
||||
let mut candidate_keys = Vec::new();
|
||||
for v in volumes {
|
||||
for p in paths {
|
||||
for direct in [false, true] {
|
||||
candidate_keys.push(key(v, p, direct));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut rng = 0x9e37_79b9_7f4a_7c15u64;
|
||||
let mut next_u64 = || {
|
||||
rng ^= rng << 13;
|
||||
rng ^= rng >> 7;
|
||||
rng ^= rng << 17;
|
||||
rng
|
||||
};
|
||||
|
||||
for step in 0..1000 {
|
||||
let op = next_u64() % 100;
|
||||
if op < 45 {
|
||||
// Insert or replace (either fresh or stale generation)
|
||||
let k = candidate_keys[(next_u64() as usize) % candidate_keys.len()].clone();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: step,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let cur_gen = cache.generation();
|
||||
let stale = (next_u64() % 5) == 0 && cur_gen > 0;
|
||||
let gen_to_use = if stale { cur_gen - 1 } else { cur_gen };
|
||||
cache.insert_if_fresh(k.clone(), Arc::clone(&entry), gen_to_use).await;
|
||||
if !stale {
|
||||
oracle.insert(k, entry);
|
||||
}
|
||||
} else if op < 65 {
|
||||
// Exact invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let p = paths[(next_u64() as usize) % paths.len()];
|
||||
cache.invalidate_exact(v, p).await;
|
||||
oracle.remove(&key(v, p, false));
|
||||
oracle.remove(&key(v, p, true));
|
||||
} else if op < 88 {
|
||||
// Prefix invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
let raw_prefix = prefixes[(next_u64() as usize) % prefixes.len()];
|
||||
cache.invalidate_under(v, raw_prefix);
|
||||
let trimmed = raw_prefix.trim_end_matches('/');
|
||||
oracle.retain(|k, _| {
|
||||
!(k.volume == v && (k.path == trimmed || k.path.strip_prefix(trimmed).is_some_and(|r| r.starts_with('/'))))
|
||||
});
|
||||
} else if op < 97 {
|
||||
// Volume invalidation
|
||||
let v = volumes[(next_u64() as usize) % volumes.len()];
|
||||
cache.invalidate_volume(v);
|
||||
oracle.retain(|k, _| k.volume != v);
|
||||
} else {
|
||||
// Full clear
|
||||
cache.clear();
|
||||
oracle.clear();
|
||||
}
|
||||
|
||||
// Verify lockstep equivalence against the reference oracle after every operation.
|
||||
for k in &candidate_keys {
|
||||
let got = cache.get(k).await;
|
||||
let expected = oracle.get(k);
|
||||
match (got, expected) {
|
||||
(Some(g), Some(e)) => assert!(
|
||||
Arc::ptr_eq(&g, e),
|
||||
"step {step}: pointer mismatch for key ({}, {}, {})",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct
|
||||
),
|
||||
(None, None) => {}
|
||||
(g, e) => panic!(
|
||||
"step {step}: oracle mismatch for key ({}, {}, {}): got={}, expected={}",
|
||||
k.volume,
|
||||
k.path,
|
||||
k.direct,
|
||||
g.is_some(),
|
||||
e.is_some()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clear and wait for maintenance to drain all entries, then verify exact counter conservation.
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear must drain all live entries and pending invalidations");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn fd_cache_concurrent_insert_and_invalidation_stress() {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
let (_directory, file) = fixture();
|
||||
let cache = Arc::new(FdCache::new());
|
||||
|
||||
// Deterministic Dekker-partner interleaving check: pause `CachedFdEntry::new`
|
||||
// at `entry_shards[shard].lock()` after it has passed the initial generation
|
||||
// check and incremented `volume_counts`, bump generation via an unmatched
|
||||
// invalidation, and verify the post-`new()` recheck refuses publication and
|
||||
// preserves the existing entry.
|
||||
let seam_key = key("v-seam", "obj-1/part.1", false);
|
||||
cache.insert(seam_key.clone(), Arc::clone(&file)).await;
|
||||
let initial_entry = cache.get(&seam_key).await.expect("initial entry must exist");
|
||||
let (vol_hash, vol_slot) = super::fd_cache_volume_hash_and_slot("v-seam");
|
||||
let entry_shard = (vol_hash as usize) & (super::FD_CACHE_PREFIX_SHARDS - 1);
|
||||
let stale_gen = cache.generation();
|
||||
let stale_entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: 999_999,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
let blocked_insert = {
|
||||
let hold = cache.tracker.entry_shards[entry_shard].lock();
|
||||
let cache_clone = Arc::clone(&cache);
|
||||
let seam_key_clone = seam_key.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
cache_clone.insert_if_fresh(seam_key_clone, stale_entry, stale_gen).await;
|
||||
});
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
|
||||
while cache.tracker.volume_counts[usize::from(vol_slot)].load(Ordering::SeqCst) < 2 {
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"insert never reached CachedFdEntry::new with the previous entry still live"
|
||||
);
|
||||
std::thread::yield_now();
|
||||
}
|
||||
cache.invalidate_under("v-seam", "unmatched-prefix");
|
||||
drop(hold);
|
||||
task
|
||||
};
|
||||
blocked_insert.await.expect("blocked insert task should complete");
|
||||
let preserved = cache
|
||||
.get(&seam_key)
|
||||
.await
|
||||
.expect("refused stale insert must not retire a valid previous entry");
|
||||
assert!(Arc::ptr_eq(&preserved, &initial_entry));
|
||||
cache.invalidate_volume("v-seam");
|
||||
|
||||
let floor_v1 = Arc::new(AtomicU64::new(0));
|
||||
let floor_v2 = Arc::new(AtomicU64::new(0));
|
||||
let mut handles = Vec::new();
|
||||
|
||||
for t in 0..8usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let file = Arc::clone(&file);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
let path = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
let k = key(vol, &path, i.is_multiple_of(2));
|
||||
let g = cache.generation();
|
||||
let entry = Arc::new(FdCacheEntry {
|
||||
file: Arc::clone(&file),
|
||||
len: g,
|
||||
invalidated: AtomicBool::new(false),
|
||||
});
|
||||
cache.insert_if_fresh(k.clone(), entry, g).await;
|
||||
let min_gen = if is_v1 {
|
||||
floor_v1.load(Ordering::SeqCst)
|
||||
} else {
|
||||
floor_v2.load(Ordering::SeqCst)
|
||||
};
|
||||
if let Some(hit) = cache.get(&k).await {
|
||||
assert!(
|
||||
hit.len >= min_gen,
|
||||
"stale descriptor served after volume invalidation: opened_gen={}, floor_gen={min_gen}",
|
||||
hit.len
|
||||
);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for t in 0..4usize {
|
||||
let cache = Arc::clone(&cache);
|
||||
let floor_v1 = Arc::clone(&floor_v1);
|
||||
let floor_v2 = Arc::clone(&floor_v2);
|
||||
handles.push(tokio::spawn(async move {
|
||||
for i in 0..250u64 {
|
||||
let is_v1 = (t + i as usize).is_multiple_of(2);
|
||||
let vol = if is_v1 { "v1" } else { "v2" };
|
||||
match i % 4 {
|
||||
0 => cache.invalidate_under(vol, &format!("dir-{}/obj-{}", t % 4, i % 16)),
|
||||
1 => {
|
||||
let p = format!("dir-{}/obj-{}/part.1", t % 4, i % 16);
|
||||
cache.invalidate_exact(vol, &p).await;
|
||||
}
|
||||
2 => {
|
||||
let before = cache.generation();
|
||||
cache.invalidate_volume(vol);
|
||||
let floor = before + 1;
|
||||
if is_v1 {
|
||||
floor_v1.fetch_max(floor, Ordering::SeqCst);
|
||||
} else {
|
||||
floor_v2.fetch_max(floor, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
_ => cache.invalidate_under("uncached-vol", "uncached/prefix"),
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
h.await.expect("worker task should succeed");
|
||||
}
|
||||
|
||||
cache.clear();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while cache.tracker.live_entries() != 0 || cache.tracker.has_pending_invalidations.load(Ordering::SeqCst) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("clear after concurrent stress must drain all live entries");
|
||||
|
||||
assert_eq!(cache.tracker.live_entries(), 0);
|
||||
assert!(cache.tracker.unindexed_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.volume_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.exact_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_counts.iter().all(|c| c.load(Ordering::SeqCst) == 0));
|
||||
assert!(cache.tracker.prefix_shards.iter().all(|s| s.lock().is_empty()));
|
||||
assert!(cache.tracker.entry_shards.iter().all(|s| s.lock().is_empty()));
|
||||
}
|
||||
|
||||
@@ -148,47 +148,58 @@ pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
native_io(move || create_dir_all_std(&path)).await
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
fn is_dir_error(path: &Path, e: &io::Error) -> bool {
|
||||
e.raw_os_error() == Some(libc::EISDIR)
|
||||
|| e.kind() == io::ErrorKind::IsADirectory
|
||||
// macOS: remove_file on a directory returns EPERM
|
||||
|| (cfg!(target_os = "macos") && e.raw_os_error() == Some(libc::EPERM))
|
||||
|| is_windows_dir_access_denied(path, e)
|
||||
}
|
||||
|
||||
/// Windows reports `remove_file` on a directory (or directory symlink) as
|
||||
/// ERROR_ACCESS_DENIED, which is indistinguishable from a real permission
|
||||
/// failure without inspecting the entry itself.
|
||||
#[cfg(windows)]
|
||||
fn is_windows_dir_access_denied(path: &Path, e: &io::Error) -> bool {
|
||||
use std::os::windows::fs::FileTypeExt;
|
||||
|
||||
e.kind() == io::ErrorKind::PermissionDenied
|
||||
&& std::fs::symlink_metadata(path).is_ok_and(|metadata| {
|
||||
let file_type = metadata.file_type();
|
||||
file_type.is_dir() || file_type.is_symlink_dir()
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || remove_std(path)).await
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || match std::fs::remove_file(&path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir_all(path)).await
|
||||
}
|
||||
Err(e) if is_dir_error(&path, &e) => std::fs::remove_dir_all(&path),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
match std::fs::remove_file(path.as_ref()) {
|
||||
let path = path.as_ref();
|
||||
match std::fs::remove_file(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => std::fs::remove_dir(path.as_ref()),
|
||||
Err(e) if is_dir_error(path, &e) => std::fs::remove_dir(path),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
+1873
-1383
File diff suppressed because it is too large
Load Diff
@@ -184,6 +184,8 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
remove_file_if_exists(&backup_path)?;
|
||||
if should_fail_local_inline_rollback_hardlink(dst_file_path) || std::fs::hard_link(dst_file_path, &backup_path).is_err() {
|
||||
let write_res = (|| -> std::io::Result<()> {
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&backup_path);
|
||||
let mut file = std::fs::File::create(&backup_path)?;
|
||||
std::io::Write::write_all(&mut file, old_metadata)?;
|
||||
if sync && !old_metadata.is_empty() {
|
||||
@@ -196,7 +198,7 @@ pub(super) fn create_local_inline_rollback_backup(
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync && !old_metadata.is_empty() {
|
||||
std::fs::File::open(&backup_path)?.sync_data()?;
|
||||
os::open_file_for_sync_std(&backup_path)?.sync_data()?;
|
||||
}
|
||||
Ok(backup_path)
|
||||
}
|
||||
@@ -367,9 +369,12 @@ fn prepare_rename_commit_and_destination_metadata_std(
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum FusedCommitAttempt {
|
||||
Committed(RenameDataResp),
|
||||
Committed(Box<RenameDataResp>),
|
||||
CommitErr(DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, Box<FileInfo>),
|
||||
#[cfg(test)]
|
||||
ResumePanic(Box<dyn std::any::Any + Send + 'static>),
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
@@ -439,29 +444,12 @@ async fn prepare_rename_commit_and_destination_metadata(
|
||||
#[derive(Debug)]
|
||||
pub(in crate::disk) struct LocalRenamePreflightRejection(());
|
||||
|
||||
impl LocalRenamePreflightRejection {
|
||||
pub(in crate::disk) fn not_dispatched() -> Self {
|
||||
Self(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct RenameDataState {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
preflight_rejection: Option<LocalRenamePreflightRejection>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl RenameDataState {
|
||||
pub(super) fn with_metadata_reservation(reservation: os::MetadataMutationReservation) -> Self {
|
||||
Self {
|
||||
metadata_reservation: Some(reservation),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep optional native MPU state machines out of ordinary rename frames.
|
||||
// Construct the future here as well as pinning it, so its temporary stays off
|
||||
// the caller's poll stack. Call only from the existing native MPU branches.
|
||||
@@ -498,20 +486,14 @@ impl LocalDisk {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_info(dst_path, &fi).await?;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
wait_for_heal_commit_test_barrier(&self.root, dst_volume, dst_path).await?;
|
||||
}
|
||||
let mut fi = fi;
|
||||
// Pre-acquired metadata must never reverse quota or multipart-source
|
||||
// lock order. These paths need their own reservation protocol.
|
||||
if state.metadata_reservation.is_some()
|
||||
&& (src_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|
||||
|| rustfs_utils::http::metadata_compat::contains_key_str(&fi.metadata, QUOTA_MUTATION_FENCE_METADATA_SUFFIX))
|
||||
{
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let pool_meta_condition = if super::pool_meta_native::is_object(&self.root, dst_volume, dst_path) {
|
||||
Some(super::pool_meta_native::PoolMetaNativeCasCondition::take(&mut fi).inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
@@ -577,7 +559,9 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let multipart_publication = if let Some(staging) = multipart_staging.as_ref() {
|
||||
Some(pin_multipart_future(|| self.prepare_native_multipart_publication(src_path, &mut fi, staging)).await?)
|
||||
Some(Arc::new(
|
||||
pin_multipart_future(|| self.prepare_native_multipart_publication_deferred(src_path, &mut fi, staging)).await?,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -605,27 +589,7 @@ impl LocalDisk {
|
||||
Some(staging) => Some(Arc::new((state.namespace_owner.take(), staging.lease.clone()))),
|
||||
None => state.namespace_owner.take(),
|
||||
};
|
||||
// A reserved lease is single-use and bound to the exact disk path.
|
||||
// Keep the independent namespace/source owner alongside it; a pre-held
|
||||
// mutex cannot be acquired again without deadlocking.
|
||||
let reservation_fence = state
|
||||
.metadata_reservation
|
||||
.as_ref()
|
||||
.and_then(|reservation| reservation.fence());
|
||||
let metadata_owner: Arc<dyn Send + Sync> = if let Some(reservation) = state.metadata_reservation.take() {
|
||||
if multipart_staging.is_some() || quota_fence_claim.is_some() {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let lease = reservation
|
||||
.into_lease_for(&root_destination_object_path, self.reservation_instance)
|
||||
.inspect_err(|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
Arc::new((lease, metadata_owner))
|
||||
} else {
|
||||
os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await
|
||||
};
|
||||
let metadata_lease = os::acquire_metadata_mutation_lease(&root_destination_object_path, metadata_owner).await;
|
||||
drop(metadata_wait);
|
||||
#[cfg(unix)]
|
||||
let parent_receipt = if native_timing {
|
||||
@@ -639,23 +603,17 @@ impl LocalDisk {
|
||||
};
|
||||
#[cfg(unix)]
|
||||
let mut receipt_mutation = native_timing
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_owner.clone()));
|
||||
.then(|| super::pool_meta_native::ReceiptMutationGuard::new(&self.pool_meta_durable_head, metadata_lease.clone()));
|
||||
let namespace_wait = os::StageTimer::new(native_timing, "pool_meta_native_namespace_lease_wait");
|
||||
let mutation_lease = os::acquire_rename_data_mutation_lease_with_owner(
|
||||
&self.root,
|
||||
dst_volume,
|
||||
&root_destination_object_path,
|
||||
staging_prefix.as_deref(),
|
||||
Some(metadata_owner),
|
||||
Some(metadata_lease),
|
||||
)
|
||||
.await;
|
||||
drop(namespace_wait);
|
||||
// The volume read lease prevents deletion of this disk's bucket until
|
||||
// publication finishes. Recheck after its wait, before any mutation.
|
||||
if reservation_fence.as_ref().is_some_and(|fence| !fence.is_valid()) {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
if let Some(claim) = quota_fence_claim {
|
||||
mutation_lease.attach_external_guard(claim);
|
||||
}
|
||||
@@ -804,11 +762,6 @@ impl LocalDisk {
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !super::has_rename_data_test_failpoints(dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
|
||||
&& !super::has_delete_version_test_failpoints()
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
@@ -852,7 +805,7 @@ impl LocalDisk {
|
||||
// requires a rollback directory; appending a new version keeps
|
||||
// an inline rollback backup for post-rename directory sync errors.
|
||||
if is_cancelled() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
@@ -875,7 +828,19 @@ impl LocalDisk {
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
if no_inline
|
||||
&& sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::is_presynced_tmp_data_dir(src_data_path))
|
||||
&& os::regular_files(src_data_path).is_ok_and(|files| files.len() >= os::PARALLEL_FILE_SYNC_THRESHOLD)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, Box::new(fi_moved)));
|
||||
}
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf.is_some(),
|
||||
@@ -894,20 +859,31 @@ impl LocalDisk {
|
||||
skip_parent = dst_parent_owned.clone();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
run_owned_file_before_open(&src_file_path_owned);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, false)?;
|
||||
if !no_inline {
|
||||
run_inline_preparation_before_backup(&dst_path_str);
|
||||
}
|
||||
|
||||
let mut staged_rollback_path = None;
|
||||
if sync_commit_metadata && let Some(ref old_metadata) = has_dst_buf {
|
||||
let backup_path =
|
||||
create_local_inline_rollback_backup(&dst_file_path_owned, &src_file_path_owned, old_metadata, true)?;
|
||||
if let Some(ref old_metadata) = has_dst_buf
|
||||
&& (sync_commit_metadata || cfg!(test))
|
||||
{
|
||||
let backup_path = create_local_inline_rollback_backup(
|
||||
&dst_file_path_owned,
|
||||
&src_file_path_owned,
|
||||
old_metadata,
|
||||
sync_commit_metadata,
|
||||
)?;
|
||||
staged_rollback_path = Some(StagedRollbackBackup(backup_path));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
|
||||
let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
let sync_result = if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
// The payload was synced, but its part.1 directory entry
|
||||
// must also be persisted before moving the data directory.
|
||||
os::fsync_dir_std(src_data_path)
|
||||
@@ -923,37 +899,124 @@ impl LocalDisk {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
} else if sync_commit_metadata && let Err(err) = prepared_metadata_source.sync_data() {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if is_cancelled() {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Err(std::io::Error::new(ErrorKind::Interrupted, "commit cancelled"));
|
||||
}
|
||||
|
||||
let guard = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
guard,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
std::fs::remove_file(&src_file_path_owned)?;
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::FileNotFound));
|
||||
}
|
||||
|
||||
if no_inline {
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
|
||||
&& let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
|
||||
{
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if has_data_dir_path_owned.is_some() {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if has_data_dir_path_owned.is_some()
|
||||
&& let Err(panic_payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
}))
|
||||
{
|
||||
return Ok(FusedCommitAttempt::ResumePanic(panic_payload));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, DiskError::Unexpected));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
) {
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
if let Some(backup_path) = staged_rollback_path.as_ref().map(StagedRollbackBackup::path) {
|
||||
let _ = remove_file_if_exists(backup_path);
|
||||
}
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
if no_inline {
|
||||
return Ok(FusedCommitAttempt::RestoreDataSource(guard, err));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
if no_inline {
|
||||
rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())?;
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
let err = rollback_inline_metadata_std(&dst_file_path_owned, None, staged_rollback_path.as_ref())
|
||||
.err()
|
||||
.unwrap_or_else(|| std::io::Error::other("test fail after metadata commit"));
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::Unexpected));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) = fsync_inline_commit_std(
|
||||
&dst_file_path_owned,
|
||||
@@ -963,8 +1026,11 @@ impl LocalDisk {
|
||||
&guard,
|
||||
)
|
||||
{
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
if no_inline {
|
||||
if let Some((_, new_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
remove_dir_all_if_exists(new_data_path)?;
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(DiskError::from(err)));
|
||||
}
|
||||
@@ -977,7 +1043,7 @@ impl LocalDisk {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
Ok(FusedCommitAttempt::Committed(RenameDataResp {
|
||||
Ok(FusedCommitAttempt::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: None,
|
||||
old_data_dir: None,
|
||||
rollback_data_dir: None,
|
||||
@@ -986,7 +1052,7 @@ impl LocalDisk {
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))
|
||||
})))
|
||||
};
|
||||
|
||||
let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
@@ -1006,7 +1072,23 @@ impl LocalDisk {
|
||||
if let Some(receipt_mutation) = receipt_mutation.as_mut() {
|
||||
receipt_mutation.complete();
|
||||
}
|
||||
return Ok(resp);
|
||||
return Ok(*resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::RestoreDataSource(directory_guard, err)) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
@@ -1014,11 +1096,269 @@ impl LocalDisk {
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
Ok(FusedCommitAttempt::ResumePanic(panic_payload)) => {
|
||||
std::panic::resume_unwind(panic_payload);
|
||||
}
|
||||
Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
|
||||
fi = restored_fi;
|
||||
fi = *restored_fi;
|
||||
(guard, dst_buf)
|
||||
}
|
||||
}
|
||||
} else if let Some(publication) = multipart_publication.as_ref()
|
||||
&& no_inline
|
||||
&& !preserve_upload_metadata
|
||||
&& !publication.legacy
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
{
|
||||
enum MultipartFusedCommitOutcome {
|
||||
Committed(Box<RenameDataResp>),
|
||||
AbortBeforeDataRename(DiskError),
|
||||
RestoreDataSource(os::RenameCommitGuard, DiskError),
|
||||
RestoreTargetAfterError(DiskError, bool),
|
||||
}
|
||||
let publication_owned = Arc::clone(publication);
|
||||
let src_parent_owned = src_file_parent.to_path_buf();
|
||||
let dst_parent_owned = dst_file_parent.to_path_buf();
|
||||
let dst_file_path_owned = dst_file_path.clone();
|
||||
let src_file_path_owned = src_file_path.clone();
|
||||
let dst_volume_dir_owned = dst_volume_dir.clone();
|
||||
let publication_root_owned = self.publication_root.clone();
|
||||
let has_data_dir_path_owned = has_data_dir_path.clone();
|
||||
let sync_data_shards = durability.syncs_data_shards();
|
||||
let sync_commit_metadata = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
let dst_volume_str = dst_volume.to_string();
|
||||
let dst_path_str = dst_path.to_string();
|
||||
let fi_moved = fi;
|
||||
#[cfg(test)]
|
||||
let root_owned = self.root.clone();
|
||||
|
||||
let outcome = os::run_blocking_namespace_operation_with_sync(
|
||||
mutation_lease.clone(),
|
||||
true,
|
||||
move || -> std::io::Result<MultipartFusedCommitOutcome> {
|
||||
let (directory_guard_owned, has_dst_buf_owned) = match prepare_rename_commit_and_destination_metadata_std(
|
||||
false,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(res) => res,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
if let Some(dst_buf) = has_dst_buf_owned.as_ref() {
|
||||
if FileMeta::is_xl2_v1_format(dst_buf)
|
||||
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||
{
|
||||
xlmeta = nmeta;
|
||||
} else {
|
||||
dst_meta_unparsable = true;
|
||||
}
|
||||
}
|
||||
let skip_parent_owned = if has_dst_buf_owned.is_some() {
|
||||
dst_parent_owned.clone()
|
||||
} else {
|
||||
dst_volume_dir_owned.clone()
|
||||
};
|
||||
let version_id = fi_moved.version_id.unwrap_or_default();
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf_owned.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_str,
|
||||
&dst_path_str,
|
||||
version_id,
|
||||
);
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
if let Some(old_data_dir) = has_old_data_dir.as_ref() {
|
||||
let _ = xlmeta.data.remove_two(version_id, *old_data_dir);
|
||||
}
|
||||
if let Err(err) = xlmeta.add_version(fi_moved) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into()));
|
||||
}
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf_owned = match xlmeta.marshal_msg() {
|
||||
Ok(buf) => buf,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err.into())),
|
||||
};
|
||||
|
||||
let prepared_metadata_source = match publication_owned.prepare_and_install_head_on_thread(
|
||||
&new_dst_buf_owned,
|
||||
has_dst_buf_owned.as_deref(),
|
||||
&dst_file_path_owned,
|
||||
&directory_guard_owned,
|
||||
) {
|
||||
Ok(source) => source,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
#[cfg(test)]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(not(test))]
|
||||
let probe_active = false;
|
||||
// When presynced, shard payloads were synced during UploadPart and
|
||||
// prepare_and_install_head_on_thread just fsynced src_data_path after
|
||||
// writing RETIREMENT_ANCHOR.
|
||||
if sync_data_shards
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref()
|
||||
&& (probe_active || !super::take_presynced_tmp_data_dir(src_data_path))
|
||||
&& let Err(err) = os::sync_dir_files_std(src_data_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
let directory_guard_owned = match super::remove_dst_base_before_commit_std(
|
||||
&dst_path_str,
|
||||
directory_guard_owned,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(err)),
|
||||
};
|
||||
if should_remove_staged_meta_before_commit(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
if let Err(err) = std::fs::remove_file(&src_file_path_owned) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into()));
|
||||
}
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::FileNotFound));
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let Some(src_parent) = src_data_path.parent() else {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::InvalidPath));
|
||||
};
|
||||
let payload_guard = match os::multipart_payload_commit_guard_std(&directory_guard_owned, src_parent) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(to_file_error(err).into())),
|
||||
};
|
||||
if let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &payload_guard)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterDataRename, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if should_fail_before_old_metadata_backup(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(
|
||||
directory_guard_owned,
|
||||
DiskError::Unexpected,
|
||||
));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterBackupBeforeMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
}
|
||||
let rename_res = if should_fail_commit_rename(&dst_path_str) {
|
||||
drop(prepared_metadata_source);
|
||||
Err(DiskError::other("test fail during metadata commit rename"))
|
||||
} else {
|
||||
os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent_owned,
|
||||
&publication_root_owned,
|
||||
&directory_guard_owned,
|
||||
)
|
||||
};
|
||||
if let Err(err) = rename_res {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreDataSource(directory_guard_owned, err));
|
||||
}
|
||||
if should_fail_after_metadata_commit(&dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(DiskError::Unexpected, false));
|
||||
}
|
||||
if crash_inject::should_crash_at(CrashPoint::RenameAfterMetaCommit, &dst_path_str) {
|
||||
return Ok(MultipartFusedCommitOutcome::AbortBeforeDataRename(DiskError::Unexpected));
|
||||
}
|
||||
if sync_commit_metadata {
|
||||
if let Some(parent) = dst_file_path_owned.parent()
|
||||
&& let Err(err) = os::fsync_commit_directory_limited_std(parent, &directory_guard_owned)
|
||||
{
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(to_file_error(err).into(), true));
|
||||
}
|
||||
if has_dst_buf_owned.is_none() {
|
||||
let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
|
||||
while let Some(dir) = ancestor {
|
||||
if !dir.starts_with(&dst_volume_dir_owned) {
|
||||
break;
|
||||
}
|
||||
if let Err(err) = os::fsync_dir_limited_std(dir) {
|
||||
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
|
||||
to_file_error(err).into(),
|
||||
true,
|
||||
));
|
||||
}
|
||||
if dir == dst_volume_dir_owned.as_path() {
|
||||
break;
|
||||
}
|
||||
ancestor = dir.parent();
|
||||
}
|
||||
}
|
||||
}
|
||||
match publication_owned.finish_on_thread(sync_commit_metadata, staging_cleanup) {
|
||||
Ok(result) => Ok(MultipartFusedCommitOutcome::Committed(Box::new(RenameDataResp {
|
||||
multipart_source: Some(Box::new(result)),
|
||||
old_data_dir: has_old_data_dir,
|
||||
rollback_data_dir: None,
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))),
|
||||
Err(err) => Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(err, false)),
|
||||
}
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?;
|
||||
|
||||
match outcome {
|
||||
MultipartFusedCommitOutcome::Committed(resp) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Ok(*resp);
|
||||
}
|
||||
MultipartFusedCommitOutcome::AbortBeforeDataRename(err) => return Err(err),
|
||||
MultipartFusedCommitOutcome::RestoreDataSource(directory_guard, err) => {
|
||||
let commit = RenameCommitContext {
|
||||
directory_guard,
|
||||
publication_root: &self.publication_root,
|
||||
mutation_lease,
|
||||
source_metadata: &src_file_path,
|
||||
destination_metadata: &dst_file_path,
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
commit.restore_published_data_source().await?;
|
||||
return Err(err);
|
||||
}
|
||||
MultipartFusedCommitOutcome::RestoreTargetAfterError(err, invalidate_before_return) => {
|
||||
if !invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
pin_multipart_future(|| publication.restore_target_after_error(mutation_lease.clone())).await?;
|
||||
if invalidate_before_return {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
prepare_rename_commit_and_destination_metadata(
|
||||
!no_inline && !preserve_upload_metadata,
|
||||
@@ -1142,13 +1482,29 @@ impl LocalDisk {
|
||||
// after the commit rename, and a crash before the rename means the
|
||||
// PUT was never acknowledged. A metadata commit: relaxed tiers
|
||||
// leave it to the page cache.
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
let probe_active = os::file_sync_probe::is_active();
|
||||
#[cfg(all(not(test), target_os = "linux"))]
|
||||
let probe_active = false;
|
||||
let shard_sync = async {
|
||||
if durability.syncs_data_shards()
|
||||
&& let Some((src_data_path, _)) = has_data_dir_path.as_ref()
|
||||
&& let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
#[cfg(target_os = "linux")]
|
||||
if !probe_active && super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
if multipart_publication.is_none()
|
||||
&& let Err(err) = os::fsync_dir_with_owner(src_data_path, Some(commit.mutation_lease.clone())).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(err) = os::sync_dir_files_with_limiter(src_data_path, self.file_sync_permits.clone()).await
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err::<(), DiskError>(to_file_error(err).into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
@@ -1241,12 +1597,13 @@ impl LocalDisk {
|
||||
);
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() {
|
||||
#[cfg(target_os = "linux")]
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
let payload_guard = if multipart_publication.is_some() {
|
||||
Some(
|
||||
os::multipart_payload_commit_guard(
|
||||
&commit.directory_guard,
|
||||
src_data_path.parent().ok_or(DiskError::InvalidPath)?,
|
||||
commit.publication_root,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await
|
||||
@@ -1255,9 +1612,9 @@ impl LocalDisk {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(target_os = "linux")]
|
||||
#[cfg(any(target_os = "linux", windows))]
|
||||
let directory_guard = payload_guard.as_ref().unwrap_or(&commit.directory_guard);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
#[cfg(not(any(target_os = "linux", windows)))]
|
||||
let directory_guard = &commit.directory_guard;
|
||||
if let Err(err) = os::rename_all_with_commit_guard(
|
||||
src_data_path,
|
||||
@@ -1540,13 +1897,22 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let multipart_source = match multipart_publication.as_ref() {
|
||||
Some(publication) => match pin_multipart_future(|| publication.finish(commit.mutation_lease.clone())).await {
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone())).await?;
|
||||
return Err(error);
|
||||
Some(publication) => {
|
||||
let target_already_synced = durability.syncs_commit_metadata();
|
||||
let staging_cleanup = multipart_staging.as_ref().map(|s| s.cleanup_handle());
|
||||
match pin_multipart_future(|| {
|
||||
publication.finish(commit.mutation_lease.clone(), target_already_synced, staging_cleanup)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => Some(Box::new(result)),
|
||||
Err(error) => {
|
||||
pin_multipart_future(|| publication.restore_target_after_error(commit.mutation_lease.clone()))
|
||||
.await?;
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
Ok(RenameDataResp {
|
||||
@@ -2187,23 +2553,8 @@ impl LocalDisk {
|
||||
dst_path: &str,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> super::super::RenameDataObservation {
|
||||
self.rename_data_observed_with_reservation(src_volume, src_path, fi, dst_volume, dst_path, (namespace_owner, None))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(in crate::disk) async fn rename_data_observed_with_reservation(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
owners: (Option<Arc<dyn Send + Sync>>, Option<os::MetadataMutationReservation>),
|
||||
) -> super::super::RenameDataObservation {
|
||||
let (namespace_owner, metadata_reservation) = owners;
|
||||
let mut state = RenameDataState {
|
||||
namespace_owner,
|
||||
metadata_reservation,
|
||||
..Default::default()
|
||||
};
|
||||
let result = self
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -421,6 +421,7 @@ impl LocalDisk {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let permit = AbortDeletePermit::new(&store, &expected, certificate.clone())?;
|
||||
super::multipart_source_state::invalidate_active_multipart_source_head(&store.source);
|
||||
let destination = self.io_get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, &Uuid::new_v4().to_string())?;
|
||||
let source = store.source.clone();
|
||||
let base = store.base.clone();
|
||||
@@ -618,6 +619,9 @@ pub(super) struct OrdinaryMultipartDeleteCheck {
|
||||
root: os::PublicationRoot,
|
||||
}
|
||||
impl OrdinaryMultipartDeleteCheck {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn validate(&self) -> Result<()> {
|
||||
let pinned = match os::pin_multipart_source_directory(&self.source, &self.base, &self.root) {
|
||||
Ok(pinned) => pinned,
|
||||
|
||||
@@ -117,6 +117,7 @@ pub(super) struct LoadedSource {
|
||||
}
|
||||
|
||||
pub(super) struct SourceStore {
|
||||
multipart_index: Arc<super::multipart_index::IndexOwner>,
|
||||
pub disk_id: Uuid,
|
||||
pub physical_slot: usize,
|
||||
pub disk_ids: Vec<Uuid>,
|
||||
@@ -186,6 +187,7 @@ impl LocalDisk {
|
||||
let logical = self.get_object_path(RUSTFS_META_MULTIPART_BUCKET, path)?;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical, owner.clone()).await;
|
||||
Ok(SourceStore {
|
||||
multipart_index: self.multipart_index.clone(),
|
||||
disk_id,
|
||||
physical_slot,
|
||||
disk_ids,
|
||||
@@ -238,9 +240,177 @@ pub(super) fn read_optional_in_pinned(
|
||||
Err(error) => Err(error.into()),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
struct FileStamp {
|
||||
len: u64,
|
||||
modified: Option<std::time::SystemTime>,
|
||||
#[cfg(unix)]
|
||||
ino: u64,
|
||||
}
|
||||
|
||||
impl FileStamp {
|
||||
fn from_path(path: &Path) -> Option<Self> {
|
||||
let meta = std::fs::symlink_metadata(path).ok()?;
|
||||
Some(Self {
|
||||
len: meta.len(),
|
||||
modified: meta.modified().ok(),
|
||||
#[cfg(unix)]
|
||||
ino: {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
meta.ino()
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ActiveMultipartSourceEntry {
|
||||
raw: Bytes,
|
||||
fi: FileInfo,
|
||||
head_stamp: FileStamp,
|
||||
unsettled_parts: std::collections::HashSet<usize>,
|
||||
parts: std::collections::HashMap<usize, (FileStamp, rustfs_filemeta::ObjectPartInfo)>,
|
||||
}
|
||||
|
||||
const ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD: usize = 64;
|
||||
|
||||
static ACTIVE_MULTIPART_SOURCE_HEADS: std::sync::LazyLock<os::ShardedPathCache<ActiveMultipartSourceEntry>> =
|
||||
std::sync::LazyLock::new(|| os::ShardedPathCache::new(ACTIVE_MULTIPART_SOURCE_HEADS_PER_SHARD));
|
||||
|
||||
pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: Bytes, fi: FileInfo) {
|
||||
let source = source.as_ref();
|
||||
let Some(head_stamp) = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) else {
|
||||
return;
|
||||
};
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.insert(
|
||||
source,
|
||||
ActiveMultipartSourceEntry {
|
||||
raw,
|
||||
fi,
|
||||
head_stamp,
|
||||
unsettled_parts: std::collections::HashSet::new(),
|
||||
parts: std::collections::HashMap::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
let (raw, fi, head_stamp) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.raw.clone(), entry.fi.clone(), entry.head_stamp))?;
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return None;
|
||||
}
|
||||
Some((raw, fi))
|
||||
}
|
||||
|
||||
pub(super) fn has_active_multipart_source_head(source: &Path) -> bool {
|
||||
let Some(head_stamp) = ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| entry.head_stamp) else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub(super) fn has_clean_active_multipart_source(source: &Path) -> bool {
|
||||
let Some((head_stamp, clean)) =
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get(source, |entry| (entry.head_stamp, entry.unsettled_parts.is_empty()))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return false;
|
||||
}
|
||||
clean
|
||||
}
|
||||
|
||||
pub(super) fn begin_active_multipart_part_transaction(source: &Path, part_number: usize) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
entry.unsettled_parts.insert(part_number);
|
||||
entry.parts.remove(&part_number);
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn record_active_multipart_part_committed(source: &Path, part: rustfs_filemeta::ObjectPartInfo) {
|
||||
let Some(data_dir) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| entry.fi.data_dir)
|
||||
.flatten()
|
||||
.filter(|dir| !dir.is_nil())
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{}.meta", part.number));
|
||||
let Some(part_stamp) = FileStamp::from_path(&part_meta_path) else {
|
||||
return;
|
||||
};
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.with_get_mut(source, |entry| {
|
||||
entry.parts.insert(part.number, (part_stamp, part));
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn finish_active_multipart_part_transaction(
|
||||
source: &Path,
|
||||
part_number: Option<usize>,
|
||||
committed: bool,
|
||||
retired_existing_txn: bool,
|
||||
) {
|
||||
let Some(part_number) = part_number.filter(|_| committed) else {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return;
|
||||
};
|
||||
if !retired_existing_txn {
|
||||
return;
|
||||
}
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove_if_mut(source, |entry| !entry.unsettled_parts.remove(&part_number));
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_part(source: &Path, part_number: usize) -> Option<rustfs_filemeta::ObjectPartInfo> {
|
||||
let (head_stamp, data_dir, expected_stamp, part) = ACTIVE_MULTIPART_SOURCE_HEADS
|
||||
.with_get(source, |entry| {
|
||||
if !entry.unsettled_parts.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let data_dir = entry.fi.data_dir.filter(|dir| !dir.is_nil())?;
|
||||
let (expected_stamp, part) = entry.parts.get(&part_number)?;
|
||||
Some((entry.head_stamp, data_dir, *expected_stamp, part.clone()))
|
||||
})
|
||||
.flatten()?;
|
||||
if !stamp_matches_active_disk(source, head_stamp) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
return None;
|
||||
}
|
||||
let part_meta_path = source.join(data_dir.to_string()).join(format!("part.{part_number}.meta"));
|
||||
if FileStamp::from_path(&part_meta_path) != Some(expected_stamp) {
|
||||
return None;
|
||||
}
|
||||
Some(part)
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_source_head(source: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.remove(source);
|
||||
}
|
||||
|
||||
pub(super) fn invalidate_active_multipart_sources_under(prefix: &Path) {
|
||||
ACTIVE_MULTIPART_SOURCE_HEADS.retain_outside_prefix(prefix);
|
||||
}
|
||||
|
||||
/// Canonical source mutations are admitted under SourceRW at the syscall,
|
||||
/// not on the earlier async caller that may have timed out.
|
||||
pub(super) fn validate_source_mutation(source: &Path, base: &Path, root: &os::PublicationRoot, initial: bool) -> Result<()> {
|
||||
if !initial && has_active_multipart_source_head(source) {
|
||||
return Ok(());
|
||||
}
|
||||
let pinned = match os::pin_multipart_source_directory(source, base, root) {
|
||||
Ok(pinned) => pinned,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
@@ -440,6 +610,7 @@ impl SourceStore {
|
||||
terminal: Option<&Path>,
|
||||
sync_source_before: bool,
|
||||
) -> std::io::Result<()> {
|
||||
invalidate_active_multipart_source_head(source);
|
||||
#[cfg(unix)]
|
||||
let mut pinned = os::pin_multipart_source_directory(source, base, root)?;
|
||||
#[cfg(not(unix))]
|
||||
@@ -470,7 +641,7 @@ impl SourceStore {
|
||||
let terminal = terminal_record_path(&source, state);
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation_with_sync(self.lease.clone(), true, move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)?;
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, false)?;
|
||||
Self::persist_state_std(&source, &base, &root, &raw, terminal.as_deref(), false)
|
||||
})
|
||||
.await
|
||||
@@ -539,7 +710,7 @@ impl SourceStore {
|
||||
let root = self.publication_root.clone();
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes)
|
||||
Self::write_owned_backup_std(&source, &base, &root, owner, target, &bytes, true)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
@@ -551,6 +722,7 @@ impl SourceStore {
|
||||
owner: Uuid,
|
||||
target: bool,
|
||||
bytes: &[u8],
|
||||
sync_source_root: bool,
|
||||
) -> std::io::Result<()> {
|
||||
let path = if target {
|
||||
target_backup_path(source, owner)
|
||||
@@ -582,9 +754,13 @@ impl SourceStore {
|
||||
if !existed {
|
||||
os::replace_multipart_source_metadata(&path, bytes, base, root)?;
|
||||
}
|
||||
os::sync_multipart_owned_record_parents(source, parent, existed, base, root)
|
||||
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
self.multipart_index
|
||||
.record_head(&format!("{}/xl.meta", self.path), bytes)
|
||||
.await?;
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
let path = self.source.join(STORAGE_FORMAT_FILE);
|
||||
let base = self.base.clone();
|
||||
let root = self.publication_root.clone();
|
||||
@@ -1097,6 +1273,82 @@ pub(super) fn load_source(
|
||||
})
|
||||
}
|
||||
|
||||
fn load_uninitialized_source(
|
||||
raw: Bytes,
|
||||
path: &str,
|
||||
disk_id: Uuid,
|
||||
physical_slot: usize,
|
||||
disk_ids: &[Uuid],
|
||||
) -> Result<LoadedSource> {
|
||||
let sha = raw_sha(&raw);
|
||||
let length = raw.len() as u64;
|
||||
let current = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(Error::other)?;
|
||||
native_source_protocol(¤t)?;
|
||||
if source_status(¤t)?.is_some()
|
||||
|| publication_identity(¤t)?.is_some()
|
||||
|| SourceConditionV1::take(&mut current.clone())?.is_some()
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let bucket = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-bucket")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let object = rustfs_utils::http::metadata_compat::get_consistent_str(¤t.metadata, "multipart-object")
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
let condition = capture_source_condition(bucket, object, path, ¤t, current.data_dir.ok_or(DiskError::FileCorrupt)?)?;
|
||||
if condition.generation.epoch != 0 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let slot = condition
|
||||
.lineage
|
||||
.geometry
|
||||
.distribution
|
||||
.iter()
|
||||
.position(|index| *index == current.erasure.index)
|
||||
.ok_or(DiskError::FileCorrupt)?;
|
||||
if slot != physical_slot || condition.lineage.geometry.disks() != disk_ids.len() {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let inventory = raw_sha(&encode(&(None::<[u8; 32]>, Some(sha), sha, None::<[u8; 32]>, None::<[u8; 32]>))?);
|
||||
let snapshot = Some(MultipartSourceSnapshot {
|
||||
disk_id,
|
||||
physical_slot: slot,
|
||||
lineage: condition.lineage,
|
||||
generation: condition.generation,
|
||||
revision: 0,
|
||||
phase: MultipartSourcePhase::Active,
|
||||
state_sha256: [0; 32],
|
||||
head_sha256: Some(sha),
|
||||
inventory_sha256: inventory,
|
||||
capability_sha256: None,
|
||||
publication: None,
|
||||
});
|
||||
Ok(LoadedSource {
|
||||
root_missing: false,
|
||||
state: None,
|
||||
head: Some(SourceHead {
|
||||
sha256: sha,
|
||||
length,
|
||||
raw: Some(raw.clone()),
|
||||
}),
|
||||
original: Some(raw),
|
||||
info: Some(current),
|
||||
snapshot,
|
||||
head_matches_state: true,
|
||||
head_matches_activation: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) const RETIREMENT_ANCHOR: &str = ".mpu-source-retirement";
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
@@ -1265,7 +1517,7 @@ impl LocalDisk {
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let _directory = os::pin_multipart_source_directory(&directory, base, root)?;
|
||||
let directory_pin = os::pin_multipart_source_directory(&directory, base, root)?;
|
||||
if !parts.is_empty() {
|
||||
let anchor_path = directory.join(RETIREMENT_ANCHOR);
|
||||
let bytes = os::read_multipart_source_metadata(&anchor_path, base, root)?;
|
||||
@@ -1298,7 +1550,13 @@ impl LocalDisk {
|
||||
}
|
||||
// An anchor-less empty DD is only rmdir'ed. A new child makes
|
||||
// this fail closed; no recursive removal can consume its bytes.
|
||||
// A Windows pin refuses DELETE sharing, so the leaf pin is
|
||||
// released first; the object pin still holds the parent.
|
||||
#[cfg(windows)]
|
||||
drop(directory_pin);
|
||||
std::fs::remove_dir(&directory)?;
|
||||
#[cfg(not(windows))]
|
||||
drop(directory_pin);
|
||||
if sync {
|
||||
os::fsync_dir_std(&object_path)?;
|
||||
}
|
||||
@@ -1421,6 +1679,8 @@ struct ValidatedPreparedPublication {
|
||||
length: u64,
|
||||
}
|
||||
|
||||
type DeferredInitialPublication = (Vec<u8>, PathBuf, Vec<u8>);
|
||||
|
||||
pub(super) struct NativeMultipartPublication {
|
||||
pub store: Arc<SourceStore>,
|
||||
pub publication: PublicationIdentity,
|
||||
@@ -1428,22 +1688,49 @@ pub(super) struct NativeMultipartPublication {
|
||||
state: parking_lot::Mutex<SourceState>,
|
||||
capability: parking_lot::Mutex<Option<MultipartUndoCapability>>,
|
||||
validated_prepared: parking_lot::Mutex<Option<ValidatedPreparedPublication>>,
|
||||
deferred_initial: parking_lot::Mutex<Option<DeferredInitialPublication>>,
|
||||
pub legacy: bool,
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(test)]
|
||||
pub(super) async fn prepare_native_multipart_publication(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, false)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_native_multipart_publication_deferred(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
self.prepare_native_multipart_publication_inner(path, incoming, staging, true)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_native_multipart_publication_inner(
|
||||
&self,
|
||||
path: &str,
|
||||
incoming: &mut FileInfo,
|
||||
staging: &MultipartCommitStaging,
|
||||
defer_persist: bool,
|
||||
) -> Result<NativeMultipartPublication> {
|
||||
// Real Complete is parts-backed, including a zero-byte last part.
|
||||
if incoming.data.is_some() || incoming.inline_data() {
|
||||
return Err(DiskError::UnsupportedDisk);
|
||||
}
|
||||
let store = Arc::new(self.multipart_store_from_staging(path, staging).await?);
|
||||
let loaded = store.load().await?;
|
||||
let loaded = if defer_persist && let Some(raw) = staging.fresh_source_head() {
|
||||
load_uninitialized_source(raw, &store.path, store.disk_id, store.physical_slot, &store.disk_ids)?
|
||||
} else {
|
||||
store.load().await?
|
||||
};
|
||||
let snapshot = loaded.snapshot.as_ref().ok_or(DiskError::ConditionalDeleteConflict)?;
|
||||
let original = loaded
|
||||
.head
|
||||
@@ -1545,20 +1832,33 @@ impl LocalDisk {
|
||||
};
|
||||
let anchor_bytes = encode(&anchor_record)?;
|
||||
state.validate(store.disk_id, &store.path)?;
|
||||
let state_raw = encode(&state)?;
|
||||
let source_dir = store.source.clone();
|
||||
let base_dir = store.base.clone();
|
||||
let pub_root = store.publication_root.clone();
|
||||
let terminal = terminal_record_path(&source_dir, &state);
|
||||
let original_bytes = original.to_vec();
|
||||
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
|
||||
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes)?;
|
||||
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
let deferred_initial = if defer_persist {
|
||||
Some((original_bytes, directory, anchor_bytes))
|
||||
} else {
|
||||
let state_raw = encode(&state)?;
|
||||
let source_dir = store.source.clone();
|
||||
let base_dir = store.base.clone();
|
||||
let pub_root = store.publication_root.clone();
|
||||
let terminal = terminal_record_path(&source_dir, &state);
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source_dir)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
|
||||
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
|
||||
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
None
|
||||
};
|
||||
let prepared_path = store.source.join(format!(".mpu-prepared-{owner}"));
|
||||
let context = NativeMultipartPublication {
|
||||
store,
|
||||
@@ -1567,6 +1867,7 @@ impl LocalDisk {
|
||||
state: parking_lot::Mutex::new(state),
|
||||
capability: parking_lot::Mutex::new(Some(capability)),
|
||||
validated_prepared: parking_lot::Mutex::new(None),
|
||||
deferred_initial: parking_lot::Mutex::new(deferred_initial),
|
||||
legacy,
|
||||
};
|
||||
Ok(context)
|
||||
@@ -1621,7 +1922,7 @@ impl NativeMultipartPublication {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)
|
||||
})
|
||||
@@ -1631,6 +1932,133 @@ impl NativeMultipartPublication {
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn prepare_and_install_head_on_thread(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let mut state = self.state.lock().clone();
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
length: new_metadata.len() as u64,
|
||||
};
|
||||
if let Some(previous) = old_metadata {
|
||||
state.previous_target_sha256 = Some(raw_sha(previous));
|
||||
state.previous_target_length = Some(previous.len() as u64);
|
||||
}
|
||||
state.prepared_sha256 = Some(validated.sha256);
|
||||
state.prepared_length = Some(validated.length);
|
||||
state.advance(MultipartSourcePhase::Prepared)?;
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
let prepared_path = &self.prepared_path;
|
||||
let canonical = self.canonical_head();
|
||||
let prepared = (|| -> std::io::Result<os::PreparedRenameSource> {
|
||||
if let Some((ref original_bytes, _, _)) = deferred_initial {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
} else {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(prepared_path, dst_file_path, directory_guard)?;
|
||||
if let Err(err) = prepared_metadata_source.write_all(new_metadata, false) {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
let setup_res: std::io::Result<()> = (|| {
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
if let Some(previous) = old_metadata {
|
||||
SourceStore::write_owned_backup_std(source, base, root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = setup_res {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
if let Err(err) = prepared_metadata_source.sync_data() {
|
||||
drop(prepared_metadata_source);
|
||||
let _ = std::fs::remove_file(prepared_path);
|
||||
return Err(err);
|
||||
}
|
||||
os::install_multipart_prepared_head(prepared_metadata_source, &canonical, base, root)
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
pub async fn prepare_and_install_head(
|
||||
&self,
|
||||
new_metadata: &[u8],
|
||||
@@ -1643,7 +2071,11 @@ impl NativeMultipartPublication {
|
||||
if !self.store.target_matches(&self.publication, new_metadata)? {
|
||||
return Err(DiskError::ConditionalDeleteConflict);
|
||||
}
|
||||
let deferred_initial = self.deferred_initial.lock().take();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let expected_lineage = state.lineage.clone();
|
||||
let expected_generation = state.generation;
|
||||
let expected_revision = state.revision;
|
||||
let validated = ValidatedPreparedPublication {
|
||||
publication: self.publication.clone(),
|
||||
sha256: raw_sha(new_metadata),
|
||||
@@ -1666,6 +2098,9 @@ impl NativeMultipartPublication {
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let backup_owner = state.backup_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let can_mark_presynced = has_active_multipart_source_head(&source)
|
||||
&& super::effective_durability(RUSTFS_META_MULTIPART_BUCKET).syncs_data_shards();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
@@ -1675,18 +2110,51 @@ impl NativeMultipartPublication {
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
let prepared = os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
if let Some((original_bytes, directory, anchor_bytes)) = deferred_initial {
|
||||
let snapshot = current
|
||||
.snapshot
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_head = current
|
||||
.head
|
||||
.as_ref()
|
||||
.and_then(SourceHead::raw)
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
if !current.head_matches_state
|
||||
|| snapshot.phase != MultipartSourcePhase::Active
|
||||
|| snapshot.capability_sha256.is_some()
|
||||
|| snapshot.lineage != expected_lineage
|
||||
|| snapshot.generation != expected_generation
|
||||
|| snapshot.revision.checked_add(1) != Some(expected_revision)
|
||||
|| current_head != original_bytes.as_slice()
|
||||
{
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, false, &original_bytes, false)?;
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
|
||||
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if can_mark_presynced {
|
||||
super::mark_presynced_tmp_data_dir(directory);
|
||||
}
|
||||
} else {
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
.ok_or_else(|| IoError::other(DiskError::ConditionalDeleteConflict))?;
|
||||
let current_raw = encode(current_state).map_err(IoError::other)?;
|
||||
if current_raw != expected_state_raw || !current.head_matches_state {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
}
|
||||
if let Some(previous) = old_bytes.as_deref() {
|
||||
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous)?;
|
||||
}
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
|
||||
#[cfg(test)]
|
||||
super::run_owned_file_before_open(&prepared_path);
|
||||
let mut prepared_metadata_source =
|
||||
@@ -1700,6 +2168,28 @@ impl NativeMultipartPublication {
|
||||
*self.validated_prepared.lock() = Some(validated);
|
||||
Ok(prepared)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub async fn prepare_and_install_head(
|
||||
self: &Arc<Self>,
|
||||
new_metadata: &[u8],
|
||||
old_metadata: Option<&[u8]>,
|
||||
dst_file_path: &Path,
|
||||
directory_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::PreparedRenameSource> {
|
||||
let this = Arc::clone(self);
|
||||
let new_bytes = new_metadata.to_vec();
|
||||
let old_bytes = old_metadata.map(|b| b.to_vec());
|
||||
let dst_file_path = dst_file_path.to_path_buf();
|
||||
let owned_directory_guard = directory_guard.clone();
|
||||
os::run_blocking_namespace_operation_with_sync(mutation_lease, true, move || {
|
||||
this.prepare_and_install_head_on_thread(&new_bytes, old_bytes.as_deref(), &dst_file_path, &owned_directory_guard)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
#[cfg(test)]
|
||||
pub async fn install_head(&self, prepared: os::PreparedRenameSource) -> Result<os::PreparedRenameSource> {
|
||||
{
|
||||
@@ -1771,7 +2261,11 @@ impl NativeMultipartPublication {
|
||||
// return an error after consuming the last claimable source heads.
|
||||
self.store.restore_exact_target_with_lease(&state, lease).await
|
||||
}
|
||||
pub async fn finish(&self, target_lease: Arc<os::NamespaceMutationLease>) -> Result<MultipartPublicationResult> {
|
||||
pub(super) fn finish_on_thread(
|
||||
&self,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let mut state = self.state.lock().clone();
|
||||
let expected_state_raw = encode(&state)?;
|
||||
let (expected_sha256, expected_length) = {
|
||||
@@ -1790,16 +2284,16 @@ impl NativeMultipartPublication {
|
||||
state.validate(self.store.disk_id, &self.store.path)?;
|
||||
let next_state_raw = encode(&state)?;
|
||||
let snapshot = snapshot_for_state(self.store.disk_id, &state, &next_state_raw, None)?;
|
||||
let source = self.store.source.clone();
|
||||
let base = self.store.base.clone();
|
||||
let root = self.store.publication_root.clone();
|
||||
let path = self.store.path.clone();
|
||||
let source = &self.store.source;
|
||||
let base = &self.store.base;
|
||||
let root = &self.store.publication_root;
|
||||
let path = &self.store.path;
|
||||
let disk_id = self.store.disk_id;
|
||||
let physical_slot = self.store.physical_slot;
|
||||
let disk_ids = self.store.disk_ids.clone();
|
||||
let terminal = terminal_record_path(&source, &state);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
let current = load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map_err(IoError::other)?;
|
||||
let disk_ids = &self.store.disk_ids;
|
||||
let terminal = terminal_record_path(source, &state);
|
||||
(|| -> std::io::Result<()> {
|
||||
let current = load_source(source, base, root, path, disk_id, physical_slot, disk_ids).map_err(IoError::other)?;
|
||||
let current_state = current
|
||||
.state
|
||||
.as_ref()
|
||||
@@ -1808,28 +2302,64 @@ impl NativeMultipartPublication {
|
||||
if current.head.is_some() || current_raw != expected_state_raw {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
&base,
|
||||
&root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
if target_already_synced {
|
||||
let pinned = os::pin_multipart_source_directory(&target_dir, base, root).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
})?;
|
||||
let target = os::read_multipart_source_metadata_in_pinned(
|
||||
&pinned,
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
if target.len() as u64 != expected_length || raw_sha(&target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
})?;
|
||||
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), true)
|
||||
})
|
||||
.await
|
||||
} else {
|
||||
os::read_and_sync_multipart_target_metadata(
|
||||
&target_dir,
|
||||
std::ffi::OsStr::new(STORAGE_FORMAT_FILE),
|
||||
base,
|
||||
root,
|
||||
|target| {
|
||||
if target.len() as u64 != expected_length || raw_sha(target) != expected_sha256 {
|
||||
return Err(IoError::other(DiskError::ConditionalDeleteConflict));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
IoError::other(DiskError::ConditionalDeleteConflict)
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
}
|
||||
SourceStore::persist_state_std(source, base, root, &next_state_raw, terminal.as_deref(), true)?;
|
||||
if let Some((io_path, cleaned)) = staging_cleanup {
|
||||
match std::fs::remove_dir(&io_path) {
|
||||
Ok(()) => cleaned.store(true, std::sync::atomic::Ordering::Release),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
|
||||
cleaned.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})()
|
||||
.map_err(DiskError::from)?;
|
||||
*self.state.lock() = state;
|
||||
let proof = self
|
||||
@@ -1842,6 +2372,21 @@ impl NativeMultipartPublication {
|
||||
undo_capability: capability,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn finish(
|
||||
self: &Arc<Self>,
|
||||
target_lease: Arc<os::NamespaceMutationLease>,
|
||||
target_already_synced: bool,
|
||||
staging_cleanup: Option<(PathBuf, Arc<std::sync::atomic::AtomicBool>)>,
|
||||
) -> Result<MultipartPublicationResult> {
|
||||
let this = Arc::clone(self);
|
||||
os::run_blocking_namespace_operation_with_sync(target_lease, true, move || {
|
||||
this.finish_on_thread(target_already_synced, staging_cleanup)
|
||||
.map_err(IoError::other)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)
|
||||
}
|
||||
}
|
||||
|
||||
impl SourceStore {
|
||||
|
||||
@@ -207,6 +207,12 @@ pub(super) struct MultipartSourceAdmission {
|
||||
initial: bool,
|
||||
}
|
||||
impl MultipartSourceAdmission {
|
||||
pub(super) fn source(&self) -> &Path {
|
||||
&self.source
|
||||
}
|
||||
pub(super) fn is_initial(&self) -> bool {
|
||||
self.initial
|
||||
}
|
||||
pub(super) fn precondition(&self) -> Arc<dyn Fn() -> std::io::Result<()> + Send + Sync> {
|
||||
let admission = self.clone();
|
||||
Arc::new(move || admission.validate())
|
||||
@@ -250,6 +256,8 @@ pub(super) struct MultipartCommitStaging {
|
||||
logical_path: PathBuf,
|
||||
io_path: PathBuf,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
cleaned: Arc<std::sync::atomic::AtomicBool>,
|
||||
fresh_source_head: Option<bytes::Bytes>,
|
||||
}
|
||||
|
||||
impl MultipartCommitStaging {
|
||||
@@ -259,10 +267,19 @@ impl MultipartCommitStaging {
|
||||
pub(super) fn private_directory(&self, directory: Uuid) -> PathBuf {
|
||||
self.io_path.join(directory.to_string())
|
||||
}
|
||||
pub(super) fn cleanup_handle(&self) -> (PathBuf, Arc<std::sync::atomic::AtomicBool>) {
|
||||
(self.io_path.clone(), Arc::clone(&self.cleaned))
|
||||
}
|
||||
pub(super) fn fresh_source_head(&self) -> Option<bytes::Bytes> {
|
||||
self.fresh_source_head.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MultipartCommitStaging {
|
||||
fn drop(&mut self) {
|
||||
if self.cleaned.load(std::sync::atomic::Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let logical_path = self.logical_path.clone();
|
||||
let io_path = self.io_path.clone();
|
||||
let owner = self.owner.clone();
|
||||
@@ -285,6 +302,14 @@ impl Drop for MultipartCommitStaging {
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
let root = upload_scope_root(upload_or_part_path, part_path);
|
||||
let Ok(source) = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, root) else {
|
||||
return false;
|
||||
};
|
||||
super::multipart_source_state::has_clean_active_multipart_source(&source)
|
||||
}
|
||||
|
||||
pub(super) fn multipart_source_admission(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -356,12 +381,14 @@ impl LocalDisk {
|
||||
let io_path = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &attempt)?;
|
||||
// The attempt root is exactly the namespace used by the tmp sweeper.
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&logical_path, owner.clone()).await;
|
||||
let staging = MultipartCommitStaging {
|
||||
let mut staging = MultipartCommitStaging {
|
||||
path: attempt,
|
||||
lease,
|
||||
logical_path,
|
||||
io_path,
|
||||
owner,
|
||||
cleaned: Arc::new(std::sync::atomic::AtomicBool::new(false)),
|
||||
fresh_source_head: None,
|
||||
};
|
||||
let canonical_metadata = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &format!("{source_path}/xl.meta"))?;
|
||||
let source_object = self.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, source_path)?;
|
||||
@@ -371,31 +398,37 @@ impl LocalDisk {
|
||||
let source_path = source_path.to_owned();
|
||||
let incoming = incoming.clone();
|
||||
let sync_directory = effective_durability(destination_volume).syncs_data_shards();
|
||||
os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
let fresh_head = os::run_cancellable_blocking_namespace_operation(Arc::clone(&staging.lease), move |cancelled| {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
let cached_head = super::multipart_source_state::get_active_multipart_source_head(&source_object);
|
||||
let (raw, source, from_active_cache) = if let Some((raw, source)) = cached_head {
|
||||
(raw, source, true)
|
||||
} else {
|
||||
let raw = os::read_multipart_source_metadata(&canonical_metadata, &base, &publication_root).map_err(|error| {
|
||||
// A retired source has no authority for this delayed completion.
|
||||
// Preserve confinement and other I/O failures as their real errors.
|
||||
if error.kind() == std::io::ErrorKind::NotFound {
|
||||
DiskError::FileNotFound.into()
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let source = get_file_info(
|
||||
&raw,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&source_path,
|
||||
"",
|
||||
FileInfoOpts {
|
||||
data: false,
|
||||
include_free_versions: false,
|
||||
include_part_checksums: true,
|
||||
},
|
||||
)
|
||||
.map_err(std::io::Error::other)?;
|
||||
(bytes::Bytes::from(raw), source, false)
|
||||
};
|
||||
let source_dir = source
|
||||
.data_dir
|
||||
.filter(|dir| !dir.is_nil())
|
||||
@@ -412,23 +445,31 @@ impl LocalDisk {
|
||||
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
|
||||
if incoming.data.is_none() && !incoming.is_remote() {
|
||||
let source_data_dir = source_object.join(source_dir.to_string());
|
||||
let pinned_source_data = (!incoming.parts.is_empty())
|
||||
.then(|| os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root))
|
||||
.transpose()?;
|
||||
let mut pinned_source_data = None;
|
||||
let mut names = Vec::with_capacity(incoming.parts.len().saturating_mul(3));
|
||||
for part in &incoming.parts {
|
||||
if cancelled() {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "multipart staging cancelled"));
|
||||
}
|
||||
let meta_name = format!("part.{}.meta", part.number);
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
let actual = rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?;
|
||||
let actual = if let Some(cached_part) =
|
||||
super::multipart_source_state::get_active_multipart_source_part(&source_object, part.number)
|
||||
{
|
||||
cached_part
|
||||
} else {
|
||||
if pinned_source_data.is_none() {
|
||||
pinned_source_data =
|
||||
Some(os::pin_multipart_source_directory(&source_data_dir, &base, &publication_root)?);
|
||||
}
|
||||
let bytes = os::read_multipart_source_metadata_in_pinned(
|
||||
pinned_source_data.as_ref().expect("pinned source data directory"),
|
||||
&source_data_dir,
|
||||
std::ffi::OsStr::new(&meta_name),
|
||||
&base,
|
||||
&publication_root,
|
||||
)?;
|
||||
rustfs_filemeta::ObjectPartInfo::unmarshal(&bytes).map_err(std::io::Error::other)?
|
||||
};
|
||||
let normalized = i64::try_from(actual.size)
|
||||
.map_err(std::io::Error::other)?
|
||||
.max(actual.actual_size);
|
||||
@@ -464,10 +505,15 @@ impl LocalDisk {
|
||||
cancelled,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
let has_state = !from_active_cache
|
||||
&& (source_object.join(".mpu-source-state").exists()
|
||||
|| source_object.join(".mpu-recovery").exists()
|
||||
|| source_object.join(".mpu-completions").exists());
|
||||
Ok((!has_state).then_some(raw))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
staging.fresh_source_head = fresh_head;
|
||||
Ok(staging)
|
||||
}
|
||||
}
|
||||
@@ -1219,4 +1265,211 @@ mod tests {
|
||||
.expect("non-part empty hash cleanup must remain supported");
|
||||
assert!(!directory.exists(), "a per-upload safety guard must not strand empty hash directories");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_source_active_head_cache_populates_and_invalidates_across_part_and_delete_paths() {
|
||||
use crate::disk::{DiskAPI as _, PartTransactionAction, ReadOptions};
|
||||
let (_root, disk, completed, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let upload = format!("{}/{}", "c".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
disk.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &upload, init_fi.clone())
|
||||
.await
|
||||
.expect("fresh multipart upload metadata");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
let read_fi = disk
|
||||
.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("cached read_version");
|
||||
assert_eq!(read_fi.data_dir, Some(data_dir));
|
||||
|
||||
let destination = format!("{upload}/{data_dir}/part.1");
|
||||
let part_meta = bytes::Bytes::from(completed.parts[0].marshal_msg().expect("part meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part transaction");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"in-flight part transaction must mark active source as having unsettled part txn"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.1",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination,
|
||||
part_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part transaction");
|
||||
assert!(
|
||||
disk.has_clean_active_multipart_source(&upload, false),
|
||||
"settled commit must restore clean active multipart source status"
|
||||
);
|
||||
let read_parts = disk
|
||||
.read_parts(RUSTFS_META_MULTIPART_BUCKET, &[format!("{destination}.meta")])
|
||||
.await
|
||||
.expect("cached read_parts");
|
||||
assert_eq!(read_parts.len(), 1);
|
||||
assert_eq!(read_parts[0].number, 1);
|
||||
|
||||
// Prepare part 2, then replay Commit for already-settled part 1: part 2's
|
||||
// pending transaction must keep the upload marked as having unsettled part txns.
|
||||
let destination2 = format!("{upload}/{data_dir}/part.2");
|
||||
let mut part2_info = completed.parts[0].clone();
|
||||
part2_info.number = 2;
|
||||
let part2_meta = bytes::Bytes::from(part2_info.marshal_msg().expect("part 2 meta"));
|
||||
disk.write_all(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
bytes::Bytes::from_static(b"immutable shard"),
|
||||
)
|
||||
.await
|
||||
.expect("stage tmp part 2");
|
||||
disk.prepare_part_transaction(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta.clone(),
|
||||
)
|
||||
.await
|
||||
.expect("prepare part 2 transaction");
|
||||
assert!(!disk.has_clean_active_multipart_source(&upload, false));
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("replayed part 1 commit settlement");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"replayed settlement for part 1 must not clear part 2's pending transaction"
|
||||
);
|
||||
disk.rename_part(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"cache-test-part/part.2",
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&destination2,
|
||||
part2_meta,
|
||||
)
|
||||
.await
|
||||
.expect("rename part 2");
|
||||
disk.settle_part_transaction(RUSTFS_META_MULTIPART_BUCKET, &destination2, PartTransactionAction::Commit)
|
||||
.await
|
||||
.expect("settle part 2 transaction");
|
||||
assert!(disk.has_clean_active_multipart_source(&upload, false));
|
||||
|
||||
// Deleting {upload}/xl.meta through delete_paths must invalidate the canonical {upload} cache key.
|
||||
disk.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[format!("{upload}/xl.meta")])
|
||||
.await
|
||||
.expect("delete xl.meta via delete_paths");
|
||||
assert!(
|
||||
!disk.has_clean_active_multipart_source(&upload, false),
|
||||
"per-file head deletion must invalidate the canonical upload cache key"
|
||||
);
|
||||
assert!(matches!(
|
||||
disk.read_version("", RUSTFS_META_MULTIPART_BUCKET, &upload, "", &ReadOptions::default())
|
||||
.await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_source_cache_publication_holds_source_lock_against_concurrent_retirement() {
|
||||
use crate::disk::{DiskAPI as _, disk_store::LocalDiskWrapper, os::prepared_publication_test_hooks as hooks};
|
||||
|
||||
let (_root, disk, _, _) = fixture().await;
|
||||
std::fs::create_dir_all(disk.io_get_bucket_path(RUSTFS_META_TMP_BUCKET).expect("TMP path")).expect("TMP bucket");
|
||||
let disk = Arc::new(disk);
|
||||
let wrapper = LocalDiskWrapper::new(Arc::clone(&disk), false);
|
||||
let upload = format!("{}/{}", "d".repeat(64), Uuid::new_v4());
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mut init_fi = FileInfo::new(&upload, 2, 2);
|
||||
init_fi.erasure.index = 1;
|
||||
init_fi.data_dir = Some(data_dir);
|
||||
init_fi.fresh = true;
|
||||
init_fi.mod_time = Some(time::OffsetDateTime::now_utc());
|
||||
|
||||
let source_dir = disk
|
||||
.io_get_object_path(RUSTFS_META_MULTIPART_BUCKET, &upload)
|
||||
.expect("source dir");
|
||||
let base_dir = disk
|
||||
.io_get_bucket_path(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.expect("multipart bucket path");
|
||||
let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let _hook = hooks::install_at(hooks::Stage::DirFsync, &source_dir, move || {
|
||||
let _ = entered_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
|
||||
let init_wrapper = wrapper.clone();
|
||||
let init_upload = upload.clone();
|
||||
let init_task = tokio::spawn(async move {
|
||||
init_wrapper
|
||||
.write_metadata("", RUSTFS_META_MULTIPART_BUCKET, &init_upload, init_fi)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), entered_rx)
|
||||
.await
|
||||
.expect("initial write_metadata reaches parent dir fsync")
|
||||
.expect("entered channel");
|
||||
|
||||
let retire_wrapper = wrapper.clone();
|
||||
let retire_path = format!("{upload}/xl.meta");
|
||||
let mut retire_fut = Box::pin(async move {
|
||||
retire_wrapper
|
||||
.delete_paths(RUSTFS_META_MULTIPART_BUCKET, &[retire_path])
|
||||
.await
|
||||
});
|
||||
assert!(
|
||||
futures::poll!(tokio::task::unconstrained(retire_fut.as_mut())).is_pending(),
|
||||
"concurrent retirement must block on SourceRW while initial head publication is in flight"
|
||||
);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
init_task.await.expect("init join").expect("initial write_metadata succeeds");
|
||||
retire_fut
|
||||
.await
|
||||
.expect("concurrent retirement completes after SourceRW release");
|
||||
|
||||
assert!(
|
||||
!super::super::multipart_source_state::has_active_multipart_source_head(&source_dir),
|
||||
"retired source must not remain active in cache after interleaved initial publication"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
super::super::multipart_source_state::validate_source_mutation(
|
||||
&source_dir,
|
||||
&base_dir,
|
||||
&disk.publication_root,
|
||||
false
|
||||
),
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"validate_source_mutation must reject non-initial mutation on retired source"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
+195
-94
@@ -27,6 +27,7 @@ mod io_admission;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod lifecycle_test_io;
|
||||
pub mod local;
|
||||
pub mod multipart_index;
|
||||
pub mod multipart_source;
|
||||
pub mod os;
|
||||
|
||||
@@ -52,6 +53,10 @@ pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart";
|
||||
pub const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
|
||||
pub const RUSTFS_META_TMP_DELETED_BUCKET: &str = ".rustfs.sys/tmp/.trash";
|
||||
pub const BUCKET_META_PREFIX: &str = "buckets";
|
||||
/// Directory under `BUCKET_META_PREFIX` that holds drive-replacement heal
|
||||
/// intents, seals and recovery proofs. It shares the bucket-metadata root with
|
||||
/// a user bucket of the same name, so bucket deletion must leave it in place.
|
||||
pub const HEAL_REPLACEMENT_RECOVERY_DIR: &str = "ahm-replacement";
|
||||
pub const FORMAT_CONFIG_FILE: &str = "format.json";
|
||||
/// Per-disk marker present while an erasure-set heal is rebuilding this disk.
|
||||
/// `LocalDisk::disk_info` reports `healing = true` while the file exists.
|
||||
@@ -106,8 +111,6 @@ pub(crate) struct RenameDataGuards {
|
||||
pub(crate) scanner_publication_lease_token: Option<Uuid>,
|
||||
pub(crate) external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
pub(crate) metadata_reservation: Option<os::MetadataMutationReservation>,
|
||||
pub(crate) metadata_reservation_token: Option<Uuid>,
|
||||
}
|
||||
|
||||
/// Local preflight evidence stays outside DiskAPI and the RPC response format.
|
||||
@@ -124,13 +127,6 @@ impl RenameDataObservation {
|
||||
}
|
||||
}
|
||||
|
||||
fn not_dispatched(error: DiskError) -> Self {
|
||||
Self {
|
||||
result: Err(error),
|
||||
preflight_rejection: Some(local::LocalRenamePreflightRejection::not_dispatched()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn rejected_before_publication(&self) -> bool {
|
||||
self.result.is_err() && self.preflight_rejection.is_some()
|
||||
}
|
||||
@@ -320,60 +316,6 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Prepare an inert single-use reservation on this disk.
|
||||
/// A lost remote response holds its bucket publication fence until expiration, but does not lock the object.
|
||||
pub async fn prepare_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Uuid> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.prepare_object_metadata(bucket, object),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.reserve_object_metadata(bucket, object, incarnation, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enqueue a prepared token in object mutation order.
|
||||
pub async fn activate_object_metadata(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
token: Uuid,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<()> {
|
||||
if timeout.is_zero() {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
match self {
|
||||
Disk::Local(disk) => tokio::time::timeout(
|
||||
timeout.min(std::time::Duration::from_secs(5)),
|
||||
disk.activate_object_metadata(bucket, object, token),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?,
|
||||
Disk::Remote(disk) => disk.activate_object_metadata(bucket, object, token, timeout).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Cancel an unclaimed reservation. False does not prove that a claim has
|
||||
/// not already started a physical mutation.
|
||||
pub async fn cancel_object_metadata_reservation(&self, token: Uuid) -> Result<bool> {
|
||||
match self {
|
||||
Disk::Local(disk) => Ok(disk.cancel_object_metadata_reservation(token)),
|
||||
Disk::Remote(disk) => disk.cancel_object_metadata_reservation(token).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Physical ownership belongs to the node hosting the local attachment.
|
||||
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
|
||||
match self {
|
||||
@@ -464,16 +406,24 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_clean_active_multipart_source(&self, upload_or_part_path: &str, part_path: bool) -> bool {
|
||||
match self {
|
||||
Self::Local(disk) => disk.has_clean_active_multipart_source(upload_or_part_path, part_path),
|
||||
Self::Remote(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner, work_budget)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(None),
|
||||
@@ -487,11 +437,19 @@ impl Disk {
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.await
|
||||
disk.try_finish_and_finalize_prepared_delete_local(
|
||||
volume,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
namespace_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(false),
|
||||
}
|
||||
@@ -568,6 +526,17 @@ impl Disk {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for Disk {
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
bucket: &str,
|
||||
marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
Self::Remote(disk) => disk.list_multipart_index_roots(bucket, marker).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.supports_guarded_delete_versions().await,
|
||||
@@ -611,6 +580,17 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.has_delete_transaction_evidence(volume, object).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
Self::Remote(disk) => disk.discover_delete_transactions_bounded(volume, object, max_entries).await,
|
||||
}
|
||||
}
|
||||
async fn discover_delete_transactions(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -623,6 +603,35 @@ impl DiskAPI for Disk {
|
||||
Self::Remote(disk) => disk.discover_delete_transactions(volume, object, cursor, limit).await,
|
||||
}
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
request: BoundedDeleteRecoverySettlement,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
Self::Remote(disk) => disk.settle_delete_transaction_bounded(volume, request, namespace_owner).await,
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
disk.inspect_delete_transaction_bounded(volume, object, transaction, max_entries)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
async fn inspect_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1307,33 +1316,18 @@ impl Disk {
|
||||
.rename_data_observed_with_guards(src_volume, src_path, fi, dst_volume, dst_path, guards)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => {
|
||||
if guards.metadata_reservation.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
if let Some(token) = guards.metadata_reservation_token {
|
||||
if guards.scanner_publication_lease_token.is_some() {
|
||||
return RenameDataObservation::not_dispatched(DiskError::InvalidPath);
|
||||
}
|
||||
return RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_with_metadata_reservation(src_volume, src_path, fi, dst_volume, dst_path, token)
|
||||
.await,
|
||||
);
|
||||
}
|
||||
RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
)
|
||||
}
|
||||
Disk::Remote(remote_disk) => RenameDataObservation::unknown(
|
||||
remote_disk
|
||||
.rename_data_borrowed_with_fence(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
guards.scanner_publication_lease_token,
|
||||
)
|
||||
.await,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1584,6 +1578,15 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
fn host_name(&self) -> String;
|
||||
fn endpoint(&self) -> Endpoint;
|
||||
async fn close(&self) -> Result<()>;
|
||||
/// Return candidates only while this disk maintains complete boot-local coverage.
|
||||
/// Unsupported or unready disks must never return an authoritative empty page.
|
||||
async fn list_multipart_index_roots(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_marker: Option<&str>,
|
||||
) -> Result<multipart_index::MultipartIndexPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
async fn get_disk_id(&self) -> Result<Option<Uuid>>;
|
||||
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()>;
|
||||
|
||||
@@ -1761,6 +1764,35 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
|
||||
) -> Result<DeleteRecoveryPage> {
|
||||
Err(DiskError::MethodNotAllowed)
|
||||
}
|
||||
/// Bounded foreground inventory owns all traversal state until return or cancellation.
|
||||
/// An implementation without the bounded protocol is an unsupported participant
|
||||
/// (the same verdict a remote peer without it receives); callers must never fall
|
||||
/// back to paginated discovery.
|
||||
async fn discover_delete_transactions_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryDiscovery> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
async fn inspect_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_object: &str,
|
||||
_transaction: Uuid,
|
||||
_max_entries: u32,
|
||||
) -> Result<BoundedDeleteRecoveryInspection> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
async fn settle_delete_transaction_bounded(
|
||||
&self,
|
||||
_volume: &str,
|
||||
_request: BoundedDeleteRecoverySettlement,
|
||||
_namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<BoundedDeleteRecoverySettlementReply> {
|
||||
Err(DiskError::UnsupportedDisk)
|
||||
}
|
||||
/// A read-only capability check; unsupported implementations must fail closed.
|
||||
async fn supports_guarded_delete_versions(&self) -> Result<()> {
|
||||
Err(DiskError::GuardedDeleteUnsupported)
|
||||
@@ -2146,6 +2178,13 @@ pub struct WalkDirOptions {
|
||||
#[serde(default)]
|
||||
pub skip_hidden_prefix_check: bool,
|
||||
|
||||
/// Bound hidden metadata read while proving a non-recursive prefix visible.
|
||||
#[serde(default)]
|
||||
pub hidden_entry_budget: usize,
|
||||
/// A prefix whose visibility was not established before the preceding cut.
|
||||
#[serde(default)]
|
||||
pub pending_prefix: Option<String>,
|
||||
|
||||
// ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found.
|
||||
pub report_notfound: bool,
|
||||
|
||||
@@ -2478,9 +2517,62 @@ pub enum DeleteRecoveryAction {
|
||||
AdoptCommit { authority: Box<DeleteRecoveryJournal> },
|
||||
}
|
||||
|
||||
/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryDiscovery {
|
||||
pub transactions: Vec<DeleteTransactionId>,
|
||||
pub visited: u32,
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
|
||||
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlement {
|
||||
pub object: String,
|
||||
pub transaction: Uuid,
|
||||
pub expected_fingerprint: DeleteRecoveryDigest,
|
||||
pub action: DeleteRecoveryAction,
|
||||
pub max_entries: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoverySettlementOutcome {
|
||||
Inspected(Box<DeleteRecoveryInspection>),
|
||||
ReceiptPublishedInspectionDeferred,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoverySettlementReply {
|
||||
pub outcome: DeleteRecoverySettlementOutcome,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct BoundedDeleteRecoveryInspection {
|
||||
pub inspection: DeleteRecoveryInspection,
|
||||
pub visited: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub enum DeleteRecoveryRpcOperation {
|
||||
SettleBounded(BoundedDeleteRecoverySettlement),
|
||||
InspectBounded {
|
||||
object: String,
|
||||
transaction: Uuid,
|
||||
max_entries: u32,
|
||||
},
|
||||
DiscoverBounded {
|
||||
object: String,
|
||||
max_entries: u32,
|
||||
},
|
||||
InspectOrphanDataDir {
|
||||
path: String,
|
||||
},
|
||||
@@ -2518,7 +2610,11 @@ impl DeleteRecoveryRpcOperation {
|
||||
/// Full metadata application and adoption share the existing mutation envelope.
|
||||
pub fn max_wire_payload_bytes(&self) -> usize {
|
||||
match self {
|
||||
Self::ApplyVersion { .. }
|
||||
Self::SettleBounded(BoundedDeleteRecoverySettlement {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
..
|
||||
})
|
||||
| Self::ApplyVersion { .. }
|
||||
| Self::ApplyVersions { .. }
|
||||
| Self::Settle {
|
||||
action: DeleteRecoveryAction::AdoptCommit { .. },
|
||||
@@ -2534,8 +2630,11 @@ impl DeleteRecoveryRpcOperation {
|
||||
pub enum DeleteRecoveryRpcReply {
|
||||
OrphanDataDir { plain: bool },
|
||||
Discovered(DeleteRecoveryPage),
|
||||
DiscoveredBounded(BoundedDeleteRecoveryDiscovery),
|
||||
Inspected(DeleteRecoveryInspection),
|
||||
InspectedBounded(BoundedDeleteRecoveryInspection),
|
||||
Settled(DeleteRecoveryInspection),
|
||||
SettledBounded(BoundedDeleteRecoverySettlementReply),
|
||||
AppliedVersion,
|
||||
AppliedVersions(Vec<Option<u32>>),
|
||||
Evidence { present: bool },
|
||||
@@ -3013,6 +3112,8 @@ mod tests {
|
||||
recursive: true,
|
||||
incl_deleted: false,
|
||||
skip_hidden_prefix_check: false,
|
||||
hidden_entry_budget: 0,
|
||||
pending_prefix: None,
|
||||
report_notfound: false,
|
||||
filter_prefix: Some("prefix_".to_string()),
|
||||
forward_to: Some("object/path".to_string()),
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! A candidate index never replaces multipart metadata or quorum validation.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||
|
||||
pub(crate) const MULTIPART_INDEX_PAGE_SIZE: usize = 256;
|
||||
/// Pages one listing accepts from one disk. Pruned pages may carry few roots, so
|
||||
/// the cursor alone no longer bounds a peer's answer; a disk that keeps
|
||||
/// continuing past this many pages contributes no coverage.
|
||||
pub(crate) const MULTIPART_INDEX_MAX_PAGES_PER_DISK: usize = 4096;
|
||||
|
||||
/// A bounded page from one disk's continuously maintained, boot-local index.
|
||||
/// `epoch` changes whenever coverage must be rebuilt; callers must not combine
|
||||
/// pages from different epochs or physical disk identities.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexPage {
|
||||
pub bucket: String,
|
||||
pub disk_id: Uuid,
|
||||
pub epoch: Uuid,
|
||||
pub roots: Vec<String>,
|
||||
pub next_marker: Option<String>,
|
||||
}
|
||||
|
||||
pub const MULTIPART_INDEX_RPC_MAX_BYTES: usize = 64 * 1024;
|
||||
pub const MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES: usize = 4096;
|
||||
|
||||
/// Additive operation inside the legacy read-only DiskInfo JSON envelope.
|
||||
/// Old peers may ignore this field, but their ordinary response cannot decode
|
||||
/// as the required signed, challenge-bound MultipartIndexRpcReply.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcRequest {
|
||||
pub disk_id: String,
|
||||
pub metrics: bool,
|
||||
pub noop: bool,
|
||||
pub multipart_index: MultipartIndexQuery,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexQuery {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub bucket: String,
|
||||
pub marker: Option<String>,
|
||||
pub page_cap: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MultipartIndexRpcReply {
|
||||
pub operation: String,
|
||||
pub version: u32,
|
||||
pub challenge: Uuid,
|
||||
pub page: MultipartIndexPage,
|
||||
pub proof: Vec<u8>,
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcRequest {
|
||||
pub fn new(bucket: &str, marker: Option<&str>) -> Self {
|
||||
Self {
|
||||
disk_id: String::new(),
|
||||
metrics: false,
|
||||
noop: true,
|
||||
multipart_index: MultipartIndexQuery {
|
||||
operation: "list_multipart_index_roots".into(),
|
||||
version: 1,
|
||||
challenge: Uuid::new_v4(),
|
||||
bucket: bucket.to_owned(),
|
||||
marker: marker.map(str::to_owned),
|
||||
page_cap: MULTIPART_INDEX_PAGE_SIZE,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> crate::disk::error::Result<()> {
|
||||
let query = &self.multipart_index;
|
||||
if !self.disk_id.is_empty()
|
||||
|| self.metrics
|
||||
|| !self.noop
|
||||
|| query.operation != "list_multipart_index_roots"
|
||||
|| query.version != 1
|
||||
|| query.challenge.is_nil()
|
||||
|| query.bucket.is_empty()
|
||||
|| query.bucket.len() > 255
|
||||
|| query.bucket.contains('/')
|
||||
|| matches!(query.bucket.as_str(), "." | "..")
|
||||
|| query.page_cap != MULTIPART_INDEX_PAGE_SIZE
|
||||
|| query.marker.as_ref().is_some_and(|marker| !valid_root(marker))
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::MethodNotAllowed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn canonical_request(&self, disk: &str) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
if disk.len() > MULTIPART_INDEX_RPC_MAX_REQUEST_BYTES {
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-request-v1", disk, self))?)
|
||||
}
|
||||
|
||||
pub fn canonical_response(&self, disk: &str, page: &MultipartIndexPage) -> crate::disk::error::Result<Vec<u8>> {
|
||||
self.validate()?;
|
||||
Ok(serde_json::to_vec(&("rustfs-multipart-index-response-v1", disk, self, page))?)
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_root(root: &str) -> bool {
|
||||
root.len() == 64 && root.bytes().all(|byte| byte.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
impl MultipartIndexPage {
|
||||
pub fn validate(&self, bucket: &str, marker: Option<&str>, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
if expected_disk.is_nil()
|
||||
|| self.disk_id != expected_disk
|
||||
|| self.epoch.is_nil()
|
||||
|| self.bucket != bucket
|
||||
|| self.roots.len() > MULTIPART_INDEX_PAGE_SIZE
|
||||
|| self
|
||||
.roots
|
||||
.iter()
|
||||
.any(|root| !valid_root(root) || marker.is_some_and(|marker| root.as_str() <= marker))
|
||||
|| self.roots.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
// The cursor names the last examined marker; pruning may leave fewer
|
||||
// live roots than examined names, so it only has to bound them.
|
||||
|| self.next_marker.as_ref().is_some_and(|next| {
|
||||
!valid_root(next)
|
||||
|| marker.is_some_and(|marker| next.as_str() <= marker)
|
||||
|| self.roots.last().is_some_and(|last| last > next)
|
||||
})
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl MultipartIndexRpcReply {
|
||||
/// This is structural validation only; the caller must also verify `proof`.
|
||||
pub fn validate(&self, request: &MultipartIndexRpcRequest, expected_disk: Uuid) -> crate::disk::error::Result<()> {
|
||||
let query = &request.multipart_index;
|
||||
request.validate()?;
|
||||
if self.operation != query.operation
|
||||
|| self.version != query.version
|
||||
|| self.challenge != query.challenge
|
||||
|| self.proof.len() != 32
|
||||
{
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
self.page.validate(&query.bucket, query.marker.as_deref(), expected_disk)
|
||||
}
|
||||
}
|
||||
|
||||
/// A disk can attest coverage only after every page belongs to the same physical
|
||||
/// identity and index generation. An interrupted/changed stream contributes no
|
||||
/// partial candidates to the erasure set's coverage count.
|
||||
pub(crate) async fn collect_pages<F, Fut>(bucket: &str, identity: Uuid, mut fetch: F) -> crate::disk::error::Result<Vec<String>>
|
||||
where
|
||||
F: FnMut(Option<String>) -> Fut,
|
||||
Fut: std::future::Future<Output = crate::disk::error::Result<MultipartIndexPage>>,
|
||||
{
|
||||
let mut epoch = None;
|
||||
let mut marker = None;
|
||||
let mut roots = Vec::new();
|
||||
let mut pages = 0_usize;
|
||||
loop {
|
||||
if pages >= MULTIPART_INDEX_MAX_PAGES_PER_DISK {
|
||||
return Err(crate::disk::error::DiskError::FileCorrupt);
|
||||
}
|
||||
pages += 1;
|
||||
let page = fetch(marker.clone()).await?;
|
||||
page.validate(bucket, marker.as_deref(), identity)?;
|
||||
if epoch.is_some_and(|epoch| epoch != page.epoch) {
|
||||
return Err(crate::disk::error::DiskError::DiskNotFound);
|
||||
}
|
||||
epoch = Some(page.epoch);
|
||||
roots.extend(page.roots);
|
||||
marker = page.next_marker;
|
||||
if marker.is_none() {
|
||||
return Ok(roots);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn multipart_index_reply_rejects_challenge_disk_cursor_and_unbounded_pages() {
|
||||
let request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let id = Uuid::new_v4();
|
||||
let mut reply = MultipartIndexRpcReply {
|
||||
operation: request.multipart_index.operation.clone(),
|
||||
version: 1,
|
||||
challenge: request.multipart_index.challenge,
|
||||
page: MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec!["a".repeat(64)],
|
||||
next_marker: None,
|
||||
},
|
||||
proof: vec![0; 32],
|
||||
};
|
||||
reply.validate(&request, id).expect("valid structural envelope");
|
||||
assert!(
|
||||
reply.validate(&MultipartIndexRpcRequest::new("bucket", None), id).is_err(),
|
||||
"old challenge cannot replay"
|
||||
);
|
||||
assert!(reply.validate(&request, Uuid::new_v4()).is_err(), "physical disk identity cannot change");
|
||||
reply.page.next_marker = Some("9".repeat(64));
|
||||
assert!(reply.validate(&request, id).is_err(), "a cursor below the last root would re-list it");
|
||||
reply.page.next_marker = Some("a".repeat(64));
|
||||
reply
|
||||
.validate(&request, id)
|
||||
.expect("a pruned short page continues from its last examined name");
|
||||
reply.page.next_marker = None;
|
||||
reply.page.roots = vec!["a".repeat(64); MULTIPART_INDEX_PAGE_SIZE + 1];
|
||||
assert!(reply.validate(&request, id).is_err(), "peer cannot enlarge the fixed page cap");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_index_response_proof_binds_the_complete_query_and_page() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let mut request = MultipartIndexRpcRequest::new("bucket", None);
|
||||
let mut page = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: Uuid::new_v4(),
|
||||
epoch: Uuid::new_v4(),
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let canonical = request.canonical_response("disk", &page).expect("canonical response");
|
||||
let proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&canonical).expect("signed response");
|
||||
let verify = |request: &MultipartIndexRpcRequest, page: &MultipartIndexPage| {
|
||||
crate::cluster::rpc::verify_tonic_rpc_response_proof(
|
||||
&request.canonical_response("disk", page).expect("canonical"),
|
||||
&proof,
|
||||
)
|
||||
};
|
||||
verify(&request, &page).expect("exact response");
|
||||
request.multipart_index.marker = Some("0".repeat(64));
|
||||
assert!(verify(&request, &page).is_err(), "cursor is signed");
|
||||
request.multipart_index.marker = None;
|
||||
page.epoch = Uuid::new_v4();
|
||||
assert!(verify(&request, &page).is_err(), "boot-local coverage epoch is signed");
|
||||
}
|
||||
#[test]
|
||||
fn multipart_index_page_cursor_bounds_roots_without_requiring_a_full_page() {
|
||||
let disk = Uuid::new_v4();
|
||||
let page = |roots: Vec<String>, next_marker: Option<String>| MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: disk,
|
||||
epoch: Uuid::new_v4(),
|
||||
roots,
|
||||
next_marker,
|
||||
};
|
||||
// Pruning leaves fewer live roots than examined names; the cursor still advances.
|
||||
page(vec![format!("{:064x}", 1)], Some(format!("{:064x}", 5)))
|
||||
.validate("bucket", None, disk)
|
||||
.expect("short page with a cursor");
|
||||
page(vec![], Some(format!("{:064x}", 5)))
|
||||
.validate("bucket", Some(&format!("{:064x}", 2)), disk)
|
||||
.expect("all examined names pruned");
|
||||
assert!(
|
||||
page(vec![format!("{:064x}", 6)], Some(format!("{:064x}", 5)))
|
||||
.validate("bucket", None, disk)
|
||||
.is_err(),
|
||||
"a root beyond the cursor would be re-listed"
|
||||
);
|
||||
assert!(
|
||||
page(vec![], Some(format!("{:064x}", 2)))
|
||||
.validate("bucket", Some(&format!("{:064x}", 2)), disk)
|
||||
.is_err(),
|
||||
"the cursor must advance past the request marker"
|
||||
);
|
||||
assert!(
|
||||
page(vec![], Some("not-a-sha".to_owned()))
|
||||
.validate("bucket", None, disk)
|
||||
.is_err(),
|
||||
"the cursor is a marker name"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_page_collection_bounds_the_pages_one_disk_may_answer() {
|
||||
let id = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
let mut served = 0_usize;
|
||||
let result = collect_pages("bucket", id, |marker| {
|
||||
served += 1;
|
||||
let next = marker.map_or(0_u64, |marker| u64::from_str_radix(&marker[48..], 16).expect("hex cursor") + 1);
|
||||
std::future::ready(Ok(MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch,
|
||||
roots: vec![],
|
||||
next_marker: Some(format!("{next:064x}")),
|
||||
}))
|
||||
})
|
||||
.await;
|
||||
assert!(matches!(result, Err(crate::disk::error::DiskError::FileCorrupt)), "{result:?}");
|
||||
assert_eq!(served, MULTIPART_INDEX_MAX_PAGES_PER_DISK, "the peer is cut off at the page cap");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_index_page_collection_rejects_replaced_disk_or_restarted_epoch() {
|
||||
let id = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
for replace_disk in [false, true] {
|
||||
let first = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: id,
|
||||
epoch,
|
||||
roots: (0..MULTIPART_INDEX_PAGE_SIZE).map(|index| format!("{index:064x}")).collect(),
|
||||
next_marker: Some(format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1)),
|
||||
};
|
||||
let second = MultipartIndexPage {
|
||||
bucket: "bucket".into(),
|
||||
disk_id: if replace_disk { Uuid::new_v4() } else { id },
|
||||
epoch: if replace_disk { epoch } else { Uuid::new_v4() },
|
||||
roots: vec![],
|
||||
next_marker: None,
|
||||
};
|
||||
let mut replies = std::collections::VecDeque::from([first, second]);
|
||||
let result = collect_pages("bucket", id, |marker| {
|
||||
let page = replies.pop_front().expect("bounded fixture pages");
|
||||
if marker.is_some() {
|
||||
assert!(page.roots.is_empty());
|
||||
}
|
||||
std::future::ready(Ok(page))
|
||||
})
|
||||
.await;
|
||||
// A replaced disk fails the page's identity check; a restarted
|
||||
// epoch fails the cross-page continuity check.
|
||||
let expected_corrupt = replace_disk;
|
||||
assert!(
|
||||
match &result {
|
||||
Err(crate::disk::error::DiskError::FileCorrupt) => expected_corrupt,
|
||||
Err(crate::disk::error::DiskError::DiskNotFound) => !expected_corrupt,
|
||||
_ => false,
|
||||
},
|
||||
"changed physical source cannot contribute partial coverage: {result:?}"
|
||||
);
|
||||
assert!(replies.is_empty(), "exercise the continuation response");
|
||||
}
|
||||
}
|
||||
}
|
||||
+1517
-632
File diff suppressed because it is too large
Load Diff
@@ -269,6 +269,11 @@ pub enum StorageError {
|
||||
/// retained for operator inspection.
|
||||
#[error("delete recovery found an ambiguous legacy residue; operator inspection is required")]
|
||||
DeleteRecoveryLegacyAmbiguous,
|
||||
/// Delete recovery exhausted the per-request work budget before it could
|
||||
/// classify every residue. Retrying later makes progress, so callers treat
|
||||
/// it like `SlowDown`; it stays distinct so the disk error round-trips.
|
||||
#[error("delete recovery work limit reached; please retry")]
|
||||
DeleteRecoveryWorkLimit,
|
||||
#[error("Decommission not started")]
|
||||
DecommissionNotStarted,
|
||||
#[error("Decommission already running")]
|
||||
@@ -480,6 +485,7 @@ impl From<DiskError> for StorageError {
|
||||
DiskError::InvalidPath => StorageError::InvalidPath,
|
||||
DiskError::ConditionalDeleteConflict => StorageError::ConditionalDeleteConflict,
|
||||
DiskError::GuardedDeleteUnsupported => StorageError::GuardedDeleteUnsupported,
|
||||
DiskError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
|
||||
DiskError::RemoteClientUnavailable(detail) => StorageError::RemoteClientUnavailable(detail),
|
||||
}
|
||||
}
|
||||
@@ -507,6 +513,7 @@ impl StorageError {
|
||||
StorageError::MethodNotAllowed => DiskError::MethodNotAllowed,
|
||||
StorageError::StorageFull => DiskError::DiskFull,
|
||||
StorageError::SlowDown => DiskError::TooManyOpenFiles,
|
||||
StorageError::DeleteRecoveryWorkLimit => DiskError::DeleteRecoveryWorkLimit,
|
||||
StorageError::MaxVersionsExceeded => DiskError::MaxVersionsExceeded,
|
||||
StorageError::InconsistentDisk => DiskError::InconsistentDisk,
|
||||
StorageError::UnsupportedDisk => DiskError::UnsupportedDisk,
|
||||
@@ -710,6 +717,7 @@ impl Clone for StorageError {
|
||||
StorageError::StorageFull => StorageError::StorageFull,
|
||||
StorageError::SlowDown => StorageError::SlowDown,
|
||||
StorageError::DeleteRecoveryLegacyAmbiguous => StorageError::DeleteRecoveryLegacyAmbiguous,
|
||||
StorageError::DeleteRecoveryWorkLimit => StorageError::DeleteRecoveryWorkLimit,
|
||||
StorageError::PrefixAccessDenied(a, b) => StorageError::PrefixAccessDenied(a.clone(), b.clone()),
|
||||
StorageError::InvalidUploadIDKeyCombination(a, b) => {
|
||||
StorageError::InvalidUploadIDKeyCombination(a.clone(), b.clone())
|
||||
@@ -852,6 +860,7 @@ impl StorageError {
|
||||
StorageError::StorageFull => StorageErrorCode::StorageFull,
|
||||
StorageError::SlowDown => StorageErrorCode::SlowDown,
|
||||
StorageError::DeleteRecoveryLegacyAmbiguous => StorageErrorCode::DeleteRecoveryLegacyAmbiguous,
|
||||
StorageError::DeleteRecoveryWorkLimit => StorageErrorCode::DeleteRecoveryWorkLimit,
|
||||
StorageError::PrefixAccessDenied(_, _) => StorageErrorCode::PrefixAccessDenied,
|
||||
StorageError::InvalidUploadIDKeyCombination(_, _) => StorageErrorCode::InvalidUploadIDKeyCombination,
|
||||
StorageError::MalformedUploadID(_) => StorageErrorCode::MalformedUploadID,
|
||||
@@ -958,6 +967,7 @@ impl StorageError {
|
||||
StorageErrorCode::StorageFull => Some(StorageError::StorageFull),
|
||||
StorageErrorCode::SlowDown => Some(StorageError::SlowDown),
|
||||
StorageErrorCode::DeleteRecoveryLegacyAmbiguous => Some(StorageError::DeleteRecoveryLegacyAmbiguous),
|
||||
StorageErrorCode::DeleteRecoveryWorkLimit => Some(StorageError::DeleteRecoveryWorkLimit),
|
||||
StorageErrorCode::PrefixAccessDenied => {
|
||||
Some(StorageError::PrefixAccessDenied(Default::default(), Default::default()))
|
||||
}
|
||||
@@ -1700,8 +1710,8 @@ mod tests {
|
||||
// covers every variant and picks up newly appended ones automatically.
|
||||
let all_variants: Vec<DiskError> = (1u32..).map_while(DiskError::from_u32).collect();
|
||||
assert!(
|
||||
all_variants.len() >= 42,
|
||||
"DiskError variant enumeration shrank: got {}, expected at least 42",
|
||||
all_variants.len() >= 46,
|
||||
"DiskError variant enumeration shrank: got {}, expected at least 46",
|
||||
all_variants.len()
|
||||
);
|
||||
|
||||
|
||||
@@ -79,6 +79,14 @@ pub fn compression_scheme_to_read_plan(scheme: &str) -> std::io::Result<(Compres
|
||||
Ok((CompressionAlgorithm::from_str(scheme)?, ReadCompressionBackend::Legacy))
|
||||
}
|
||||
|
||||
/// Validate that replicated compression metadata can be read by this build.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error for an unsupported compression scheme.
|
||||
pub fn validate_compression_scheme(scheme: &str) -> std::io::Result<()> {
|
||||
compression_scheme_to_read_plan(scheme).map(|_| ())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ReadEncryptionBackend {
|
||||
Legacy,
|
||||
@@ -429,6 +437,7 @@ impl WritePlan {
|
||||
let checksum = reader.content_hash().clone();
|
||||
let trailer = reader.get_trailer().cloned();
|
||||
|
||||
let delegate_inner_etag = self.compression.is_none() && reader.is_etag_reader();
|
||||
let encrypted = self.encryption.is_some();
|
||||
if let Some(algorithm) = self.compression {
|
||||
reader = HashReader::from_reader(
|
||||
@@ -443,12 +452,15 @@ impl WritePlan {
|
||||
|
||||
if let Some(encryption) = self.encryption {
|
||||
let etag_nonce = match encryption.mode {
|
||||
WriteEncryptionMode::Singlepart { base_nonce } | WriteEncryptionMode::MultipartLegacy { base_nonce, .. } => {
|
||||
base_nonce
|
||||
}
|
||||
WriteEncryptionMode::SinglepartObjectKey | WriteEncryptionMode::MultipartObjectKey { .. } => [0u8; 12],
|
||||
WriteEncryptionMode::Singlepart { base_nonce } => base_nonce,
|
||||
WriteEncryptionMode::SinglepartObjectKey => [0u8; 12],
|
||||
// Each part write needs an independent fingerprint, including a
|
||||
// retry at the same part number. This salt affects only the ETag
|
||||
// MAC; encryption framing and its data nonce stay unchanged.
|
||||
WriteEncryptionMode::MultipartLegacy { .. } | WriteEncryptionMode::MultipartObjectKey { .. } => rand::random(),
|
||||
};
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?;
|
||||
let protected_reader = rustfs_rio::EtagReader::new_encrypted(reader, encryption.key_bytes, etag_nonce)?
|
||||
.with_inner_etag_delegation(delegate_inner_etag);
|
||||
reader = match encryption.mode {
|
||||
WriteEncryptionMode::SinglepartObjectKey => HashReader::from_reader(
|
||||
#[cfg(feature = "rio-v2")]
|
||||
@@ -556,6 +568,54 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_etags_are_unique_per_write_without_changing_data_encryption() {
|
||||
let plaintext = b"same confidential part";
|
||||
let size = i64::try_from(plaintext.len()).expect("test length");
|
||||
for object_key_mode in [false, true] {
|
||||
let mut tags = std::collections::HashSet::new();
|
||||
for part_number in [1, 2, 1] {
|
||||
let encryption = if object_key_mode {
|
||||
WriteEncryption::multipart_object_key([7; 32], part_number)
|
||||
} else {
|
||||
WriteEncryption::multipart([7; 32], [9; 12], part_number as usize)
|
||||
};
|
||||
let source = HashReader::from_stream(Cursor::new(plaintext), size, size, None, None, false).expect("plaintext");
|
||||
let mut reader = WritePlan::new()
|
||||
.with_encryption(encryption)
|
||||
.apply(source, size)
|
||||
.expect("part plan");
|
||||
let mut ciphertext = Vec::new();
|
||||
reader.read_to_end(&mut ciphertext).await.expect("encrypt part");
|
||||
let tag = reader.try_resolve_etag().expect("part fingerprint");
|
||||
assert_eq!(reader.try_resolve_etag().as_ref(), Some(&tag), "reading a stored result stays stable");
|
||||
assert!(
|
||||
tags.insert(tag),
|
||||
"different part numbers and retries must not identify matching plaintext"
|
||||
);
|
||||
#[cfg(feature = "rio-v2")]
|
||||
let mut decrypt = if object_key_mode {
|
||||
DecryptReader::new_multipart_with_object_key(Cursor::new(ciphertext), [7; 32], vec![part_number as usize])
|
||||
} else {
|
||||
DecryptReader::new_multipart(Cursor::new(ciphertext), [7; 32], [9; 12], vec![part_number as usize])
|
||||
};
|
||||
#[cfg(not(feature = "rio-v2"))]
|
||||
let mut decrypt = DecryptReader::new_multipart(
|
||||
Cursor::new(ciphertext),
|
||||
[7; 32],
|
||||
if object_key_mode { [0; 12] } else { [9; 12] },
|
||||
vec![part_number as usize],
|
||||
);
|
||||
let mut decoded = Vec::new();
|
||||
decrypt
|
||||
.read_to_end(&mut decoded)
|
||||
.await
|
||||
.expect("existing data nonce still decrypts");
|
||||
assert_eq!(decoded, plaintext);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_non_trailing_checksum_survives(plan: WritePlan) {
|
||||
let plaintext = b"checksum-context-through-write-plan".repeat(256);
|
||||
let actual_size = plaintext.len() as i64;
|
||||
|
||||
@@ -333,6 +333,34 @@ struct ProofCache {
|
||||
preferred: usize,
|
||||
}
|
||||
|
||||
type VerifiedStripeKey = (uuid::Uuid, [u8; 32], u32);
|
||||
|
||||
static VERIFIED_STRIPE_RECORDS: std::sync::LazyLock<parking_lot::Mutex<std::collections::HashMap<VerifiedStripeKey, Bytes>>> =
|
||||
std::sync::LazyLock::new(|| parking_lot::Mutex::new(std::collections::HashMap::new()));
|
||||
|
||||
fn get_verified_stripe_record(part: &PartIntegrity, stripe: u32) -> Option<Bytes> {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.get(&(part.generation, part.root, stripe))
|
||||
.cloned()
|
||||
}
|
||||
|
||||
fn put_verified_stripe_record(part: &PartIntegrity, stripe: u32, record: Bytes) {
|
||||
let mut map = VERIFIED_STRIPE_RECORDS.lock();
|
||||
if map.len() >= 4096
|
||||
&& let Some(evict_key) = map.keys().next().copied()
|
||||
{
|
||||
map.remove(&evict_key);
|
||||
}
|
||||
map.insert((part.generation, part.root, stripe), record);
|
||||
}
|
||||
|
||||
pub(crate) fn invalidate_verified_stripe_records(generation: uuid::Uuid) {
|
||||
VERIFIED_STRIPE_RECORDS
|
||||
.lock()
|
||||
.retain(|(stored_gen, _, _), _| *stored_gen != generation);
|
||||
}
|
||||
|
||||
impl PartProofReader {
|
||||
pub(crate) async fn reconstruction_proof(
|
||||
&self,
|
||||
@@ -410,6 +438,9 @@ impl PartProofReader {
|
||||
|
||||
pub(crate) async fn record(&self, stripe: u32) -> io::Result<Bytes> {
|
||||
let offset = self.part.record_offset(stripe).map_err(|_| corrupt())?;
|
||||
if let Some(bytes) = get_verified_stripe_record(&self.part, stripe) {
|
||||
return Ok(bytes);
|
||||
}
|
||||
let mut cached = self.cached.lock().await;
|
||||
if let Some((_, bytes)) = cached.records.iter().find(|(number, _)| *number == stripe) {
|
||||
return Ok(bytes.clone());
|
||||
@@ -465,6 +496,9 @@ impl PartProofReader {
|
||||
bytes
|
||||
};
|
||||
self.part.verify_record(stripe, &record).map_err(|_| corrupt())?;
|
||||
if self.part.stripes <= 4 {
|
||||
put_verified_stripe_record(&self.part, stripe, record.clone());
|
||||
}
|
||||
if cached.records.len() == 2 {
|
||||
cached.records.pop_front();
|
||||
}
|
||||
@@ -863,4 +897,41 @@ mod tests {
|
||||
prepared.part.verify_shard(stripe, 15, b"x", &record).expect("spilled proof");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verified_stripe_record_cache_serves_verified_record_and_still_verifies_payload() {
|
||||
let layout = IntegrityLayout::new(2, 2, 8, false).expect("layout");
|
||||
let mut builder = IntegrityBuilder::new(layout, 1).expect("builder");
|
||||
let shards = [b"ABCD".as_slice(), b"EFGH", b"IJKL", b"MNOP"];
|
||||
builder.push(shards.into_iter()).await.expect("stripe");
|
||||
let prepared = builder.finish(8).await.expect("index");
|
||||
let inline_bytes = prepared.inline_bytes().expect("inline bytes");
|
||||
let part = prepared.part;
|
||||
|
||||
let warm_reader = PartProofReader {
|
||||
part: part.clone(),
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: Some(inline_bytes),
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
};
|
||||
warm_reader.verify(0, 0, shards[0]).await.expect("warm verify");
|
||||
assert!(
|
||||
get_verified_stripe_record(&part, 0).is_some(),
|
||||
"small verified stripe record should populate the verified stripe cache"
|
||||
);
|
||||
|
||||
let cached_reader = Arc::new(PartProofReader {
|
||||
part,
|
||||
sources: Vec::new(),
|
||||
volume: "bucket".to_owned(),
|
||||
inline: None,
|
||||
cached: Mutex::new(ProofCache::default()),
|
||||
});
|
||||
cached_reader.verify(0, 0, shards[0]).await.expect("cached verify");
|
||||
assert!(
|
||||
cached_reader.verify(0, 0, shards[1]).await.is_err(),
|
||||
"cached record must still reject corrupt or mismatched shard payloads"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,20 +160,6 @@ pub struct ObjectLockConfigSnapshot {
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
pub(crate) fn owns_lifecycle_guard(&self) -> bool {
|
||||
self._lifecycle_guard.is_some()
|
||||
}
|
||||
|
||||
pub(crate) fn owns_lifecycle_guard_for(&self, bucket: &str, incarnation: Option<Uuid>) -> bool {
|
||||
self.bucket.as_deref() == Some(bucket)
|
||||
&& incarnation.is_some()
|
||||
&& self.bucket_incarnation_id == incarnation
|
||||
&& self
|
||||
._lifecycle_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| !guard.is_lock_lost() && !guard.is_released())
|
||||
}
|
||||
|
||||
pub(crate) fn new(state: crate::bucket::metadata_sys::ObjectLockConfigState) -> Self {
|
||||
Self {
|
||||
store_id: None,
|
||||
@@ -963,8 +949,82 @@ pub enum ShardIntegrityWriteMode {
|
||||
Protected,
|
||||
}
|
||||
|
||||
/// Shared foreground recovery allowance. Clones of ObjectOptions retain one request budget.
|
||||
/// Credits cover directory entries (including a minimum charge for an empty probe) and
|
||||
/// transaction inspections. Each bounded call is admitted while credits remain and is
|
||||
/// charged for the entries it visited when it completes.
|
||||
#[derive(Debug)]
|
||||
pub struct DeleteRecoveryWorkBudget {
|
||||
remaining: std::sync::atomic::AtomicU32,
|
||||
}
|
||||
|
||||
/// Entries one disk may visit per recovery step (discovery plus inspection).
|
||||
/// Every disk of every set a request fans out to is admitted up to this much
|
||||
/// and charged on completion, so a budget must cover that many disks.
|
||||
const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
|
||||
|
||||
impl Default for DeleteRecoveryWorkBudget {
|
||||
/// Covers a full 16-drive set with headroom; `for_disks` scales it when a
|
||||
/// request spans more drives than that.
|
||||
fn default() -> Self {
|
||||
Self::new(16 * 1024 * 1024)
|
||||
}
|
||||
}
|
||||
|
||||
impl DeleteRecoveryWorkBudget {
|
||||
/// A per-request budget sized for `disks` concurrently admitted recovery steps.
|
||||
pub(crate) fn for_disks(disks: usize) -> Self {
|
||||
let per_disk = u64::from(DELETE_RECOVERY_RESERVATION_PER_DISK) * 2;
|
||||
let scaled = u64::try_from(disks).unwrap_or(u64::MAX).saturating_mul(per_disk);
|
||||
let default = u64::from(Self::default().remaining());
|
||||
Self::new(u32::try_from(scaled.max(default)).unwrap_or(u32::MAX))
|
||||
}
|
||||
|
||||
pub(crate) fn new(entries: u32) -> Self {
|
||||
Self {
|
||||
remaining: std::sync::atomic::AtomicU32::new(entries),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reserve(&self, maximum: u32) -> crate::disk::error::Result<u32> {
|
||||
use std::sync::atomic::Ordering;
|
||||
self.remaining
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
|
||||
(remaining > 0).then(|| remaining.saturating_sub(maximum))
|
||||
})
|
||||
.map(|remaining| remaining.min(maximum))
|
||||
.map_err(|_| crate::disk::error::DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
|
||||
pub(crate) fn remaining(&self) -> u32 {
|
||||
self.remaining.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
/// Admit one bounded disk call: refuse once the budget is spent, otherwise
|
||||
/// cap the call at `maximum` or what remains. The work actually performed is
|
||||
/// charged on completion with `charge`, so concurrent calls never exhaust
|
||||
/// the budget by reservation alone (a bulk delete fans out dozens of
|
||||
/// inspections per key); each call is still bounded by its own cap.
|
||||
pub(crate) fn admit(&self, maximum: u32) -> crate::disk::error::Result<u32> {
|
||||
match self.remaining() {
|
||||
0 => Err(crate::disk::error::DiskError::DeleteRecoveryWorkLimit),
|
||||
remaining => Ok(remaining.min(maximum)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Record the entries a bounded call visited (at least one per call).
|
||||
pub(crate) fn charge(&self, visited: u32) {
|
||||
use std::sync::atomic::Ordering;
|
||||
let _ = self.remaining.try_update(Ordering::AcqRel, Ordering::Acquire, |remaining| {
|
||||
Some(remaining.saturating_sub(visited.max(1)))
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
#[doc(hidden)]
|
||||
pub delete_recovery_work_budget: Option<Arc<DeleteRecoveryWorkBudget>>,
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
pub max_parity: bool,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
@@ -1116,6 +1176,12 @@ pub struct ObjectOptions {
|
||||
pub expiration: ExpirationOptions,
|
||||
pub lifecycle_audit_event: LcAuditEvent,
|
||||
|
||||
/// In-process request gate evaluated on authoritative completion metadata.
|
||||
/// Not persisted or sent over RPC. The trait deliberately has no Debug bound:
|
||||
/// implementations may own customer encryption keys.
|
||||
#[doc(hidden)]
|
||||
pub multipart_completion_authorization:
|
||||
Option<Arc<dyn crate::storage_api_contracts::multipart::MultipartCompletionAuthorization<Error = Error>>>,
|
||||
pub eval_metadata: Option<HashMap<String, String>>,
|
||||
/// Internal compare-and-set condition for replication workers publishing
|
||||
/// terminal status after remote I/O. Storage validates it while holding
|
||||
@@ -1129,6 +1195,10 @@ pub struct ObjectOptions {
|
||||
/// Authoritative bucket Object Lock snapshot installed inside `ECStore`
|
||||
/// before a destructive commit reaches the set layer.
|
||||
pub object_lock_config_snapshot: Option<Arc<ObjectLockConfigSnapshot>>,
|
||||
/// ECStore single PUTs acquire their destructive-write configuration guards
|
||||
/// only after staging the client body. External lock owners supply a snapshot.
|
||||
#[doc(hidden)]
|
||||
pub defer_put_object_lock_snapshot: bool,
|
||||
|
||||
pub want_checksum: Option<Checksum>,
|
||||
pub skip_verify_bitrot: bool,
|
||||
@@ -1160,6 +1230,11 @@ pub enum ReplicationStatusWritebackMode {
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
pub(crate) fn ensure_delete_recovery_work_budget(&mut self) {
|
||||
self.delete_recovery_work_budget
|
||||
.get_or_insert_with(|| Arc::new(DeleteRecoveryWorkBudget::default()));
|
||||
}
|
||||
|
||||
pub(crate) fn shard_integrity_write_enabled(&self) -> bool {
|
||||
match self.shard_integrity_write_mode {
|
||||
Some(ShardIntegrityWriteMode::Protected) => true,
|
||||
|
||||
@@ -283,6 +283,17 @@ impl ECStore {
|
||||
arrived.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
tokio::select! {
|
||||
// Stop must drain this pause before taking the activation writer.
|
||||
_ = cancel.cancelled() => return Err(Error::OperationCanceled),
|
||||
result = data_movement::e2e_barrier::wait(
|
||||
"rebalance",
|
||||
bucket.as_str(),
|
||||
entry.name.as_str(),
|
||||
data_movement::e2e_barrier::STAGE_AFTER_ENTRY_ADMISSION,
|
||||
) => result.map_err(Error::from)?,
|
||||
}
|
||||
let lock_lost_signal = run_guard.lock_lost_signal();
|
||||
#[cfg(test)]
|
||||
let _run_signal_test_fence = lock_lost_signal
|
||||
@@ -1899,6 +1910,10 @@ mod tests {
|
||||
&ObjectOptions {
|
||||
version_id: Some(version_id.to_string()),
|
||||
mod_time: earlier(version.mod_time),
|
||||
// Without the object lock the rename waits for every
|
||||
// drive, so no early-ACK tail is still writing the copies
|
||||
// removed below.
|
||||
no_lock: true,
|
||||
..versioned
|
||||
},
|
||||
))
|
||||
|
||||
@@ -318,6 +318,21 @@ fn is_rebalance_transient_lock_error(err: &rustfs_lock::LockError) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_metadata_admission_retryable(err: &Error) -> bool {
|
||||
match rebalance_error_source(err) {
|
||||
Error::Lock(err) => is_rebalance_transient_lock_error(err),
|
||||
// No metadata was written when lock admission failed. Each attempt
|
||||
// must acquire its quorum again before loading or saving this run.
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket,
|
||||
object,
|
||||
..
|
||||
} => bucket == crate::disk::RUSTFS_META_BUCKET && object == REBAL_META_NAME,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_rebalance_transient_io_error(err: &std::io::Error) -> bool {
|
||||
if err.kind() == std::io::ErrorKind::TimedOut {
|
||||
return true;
|
||||
@@ -385,9 +400,7 @@ where
|
||||
match result {
|
||||
Ok(value) => return Ok(value),
|
||||
Err(err) => {
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1)
|
||||
|| !matches!(rebalance_error_source(&err), Error::Lock(lock_err) if is_rebalance_transient_lock_error(lock_err))
|
||||
{
|
||||
if attempt.saturating_add(1) >= max_attempts.max(1) || !is_rebalance_metadata_admission_retryable(&err) {
|
||||
return Err(err);
|
||||
}
|
||||
#[cfg(test)]
|
||||
@@ -719,6 +732,82 @@ mod error_source_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_recovers_after_write_quorum_failure() {
|
||||
let mut attempts = 0;
|
||||
retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(if attempts == 1 {
|
||||
resolve_rebalance_meta_save_result(
|
||||
Err(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)),
|
||||
"stop_rebalance",
|
||||
)
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.expect("a later quorum-protected admission must be allowed to succeed");
|
||||
assert_eq!(attempts, 2);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_retains_persistent_write_quorum_failure() {
|
||||
for max_attempts in [0, 1, 3] {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, max_attempts, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)))
|
||||
})
|
||||
.await;
|
||||
let error = result.expect_err("exhausted admission attempts must never acknowledge a write");
|
||||
assert_eq!(attempts, max_attempts.max(1));
|
||||
assert!(matches!(
|
||||
rebalance_error_source(&error),
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rebalance_metadata_retry_stops_after_admission_is_cancelled() {
|
||||
let mut attempts = 0;
|
||||
let result = retry_rebalance_metadata_access(None, 3, || {
|
||||
attempts += 1;
|
||||
std::future::ready(Err::<(), _>(if attempts == 1 {
|
||||
rebalance_meta_lock_error(
|
||||
rustfs_lock::LockError::QuorumNotReached {
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
"write",
|
||||
)
|
||||
} else {
|
||||
Error::OperationCanceled
|
||||
}))
|
||||
})
|
||||
.await;
|
||||
assert_eq!(attempts, 2, "a stale or cancelled run cannot enter another admission attempt");
|
||||
assert!(matches!(result, Err(Error::OperationCanceled)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_metadata_retry_does_not_retry_permanent_or_untyped_errors() {
|
||||
for err in [
|
||||
@@ -731,6 +820,20 @@ mod error_source_tests {
|
||||
required: 3,
|
||||
achieved: 2,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: "data-bucket".to_string(),
|
||||
object: REBAL_META_NAME.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "write",
|
||||
bucket: crate::disk::RUSTFS_META_BUCKET.to_string(),
|
||||
object: "other-metadata.bin".to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
},
|
||||
Error::other("stale rebalance run rejected: lock acquisition timed out"),
|
||||
Error::other("rebalance distributed run fence lost"),
|
||||
Error::Io(std::io::Error::from(std::io::ErrorKind::TimedOut)),
|
||||
|
||||
@@ -786,7 +786,7 @@ impl WarmBackend for MockWarmBackend {
|
||||
bytes.len()
|
||||
};
|
||||
|
||||
Ok(tokio::io::BufReader::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
|
||||
Ok(ReadCloser::new(Cursor::new(bytes[start.min(bytes.len())..end].to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, object: &str, rv: &str) -> Result<(), std::io::Error> {
|
||||
|
||||
@@ -37,7 +37,6 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::BufReader;
|
||||
use tokio::{
|
||||
select,
|
||||
sync::{Notify, RwLock},
|
||||
@@ -8183,7 +8182,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
|
||||
@@ -8248,7 +8247,7 @@ mod tests {
|
||||
_opts: WarmBackendGetOpts,
|
||||
) -> std::result::Result<ReadCloser, std::io::Error> {
|
||||
if self.healthy {
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
} else {
|
||||
Err(std::io::Error::other("mock get failed"))
|
||||
}
|
||||
@@ -9895,7 +9894,7 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn get(&self, _object: &str, _rv: &str, _opts: WarmBackendGetOpts) -> io::Result<ReadCloser> {
|
||||
Ok(BufReader::new(Cursor::new(b"RustFS".to_vec())))
|
||||
Ok(ReadCloser::new(Cursor::new(b"RustFS".to_vec())))
|
||||
}
|
||||
|
||||
async fn remove(&self, _object: &str, _rv: &str) -> io::Result<()> {
|
||||
|
||||
@@ -2538,7 +2538,6 @@ pub(in crate::set_disk) struct RenameDataCommit {
|
||||
pub(in crate::set_disk) old_current_source: Option<FileInfo>,
|
||||
pub(in crate::set_disk) committed_file_info: FileInfo,
|
||||
pub(in crate::set_disk) tail_drain: Option<tokio::task::JoinHandle<Option<RenameTailOutcome>>>,
|
||||
pub(in crate::set_disk) tail_has_metadata_reservations: bool,
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) struct RenameTailCleanup {
|
||||
@@ -3469,8 +3468,6 @@ pub(in crate::set_disk) struct RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: Option<RenameRollbackReceipt>,
|
||||
multipart_proof_authority: Option<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
multipart_commit_owner: Option<Arc<MultipartLogicalGuards>>,
|
||||
reservation_ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
prepared_reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
}
|
||||
|
||||
impl<'a> RenameDataFenceOptions<'a> {
|
||||
@@ -3486,8 +3483,6 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
rollback_receipt: None,
|
||||
multipart_proof_authority: None,
|
||||
multipart_commit_owner: None,
|
||||
reservation_ctx: None,
|
||||
prepared_reservations: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3524,165 +3519,6 @@ impl<'a> RenameDataFenceOptions<'a> {
|
||||
self.namespace_commit_guard = namespace_commit_guard;
|
||||
self
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn with_reservation_context(
|
||||
mut self,
|
||||
ctx: Option<Arc<crate::runtime::instance::InstanceContext>>,
|
||||
reservations: Option<Vec<Option<RenameMetadataReservation>>>,
|
||||
) -> Self {
|
||||
self.reservation_ctx = ctx;
|
||||
self.prepared_reservations = reservations;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct RenameMetadataReservation {
|
||||
disk: DiskStore,
|
||||
token: Option<Uuid>,
|
||||
prepared_at: tokio::time::Instant,
|
||||
}
|
||||
|
||||
impl RenameMetadataReservation {
|
||||
async fn cancel(&mut self) -> disk::error::Result<()> {
|
||||
let Some(token) = self.token else { return Ok(()) };
|
||||
match tokio::time::timeout(Duration::from_secs(5), self.disk.cancel_object_metadata_reservation(token)).await {
|
||||
Ok(Ok(true)) => {
|
||||
self.token = None;
|
||||
Ok(())
|
||||
}
|
||||
Ok(Ok(false)) => Err(DiskError::DiskOngoingReq),
|
||||
Ok(Err(error)) => Err(error),
|
||||
Err(_) => Err(DiskError::Timeout),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RenameMetadataReservation {
|
||||
fn drop(&mut self) {
|
||||
let Some(token) = self.token.take() else { return };
|
||||
if let Ok(runtime) = tokio::runtime::Handle::try_current() {
|
||||
let disk = self.disk.clone();
|
||||
runtime.spawn(async move {
|
||||
let _ = disk.cancel_object_metadata_reservation(token).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn prepare_rename_metadata_reservations(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
incarnation: Uuid,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
owns_bucket_fence: bool,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
let prepared: Vec<disk::error::Result<Option<RenameMetadataReservation>>> = join_all(disks.iter().map(|disk| async move {
|
||||
let Some(disk) = disk else { return Ok(None) };
|
||||
if disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, disk, None, false).await?;
|
||||
}
|
||||
let prepared_at = tokio::time::Instant::now();
|
||||
let token = if disk.is_local() && owns_bucket_fence {
|
||||
let store = crate::bucket::metadata_sys::object_store_if_initialized_in(ctx)
|
||||
.await
|
||||
.ok_or(DiskError::DiskOngoingReq)?;
|
||||
store
|
||||
.reserve_local_object_metadata_at_incarnation(
|
||||
&disk.endpoint().to_string(),
|
||||
bucket,
|
||||
object,
|
||||
incarnation,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
disk.prepare_object_metadata(bucket, object, incarnation, Duration::from_secs(5))
|
||||
.await?
|
||||
};
|
||||
// No await separates receipt from installing the cancellation owner.
|
||||
Ok(Some(RenameMetadataReservation {
|
||||
disk: disk.clone(),
|
||||
token: Some(token),
|
||||
prepared_at,
|
||||
}))
|
||||
}))
|
||||
.await;
|
||||
let mut reservations = Vec::with_capacity(prepared.len());
|
||||
let mut failed = false;
|
||||
for result in prepared {
|
||||
match result {
|
||||
Ok(reservation) => reservations.push(reservation),
|
||||
Err(_) => {
|
||||
failed = true;
|
||||
reservations.push(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
if failed {
|
||||
// Prepared tokens are inert; dropping them cannot leave a queued writer.
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
pub(crate) fn align_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
disks: &[Option<DiskStore>],
|
||||
) -> Option<Vec<Option<RenameMetadataReservation>>> {
|
||||
if reservations.len() != disks.len() {
|
||||
return None;
|
||||
}
|
||||
let mut aligned = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let Some(disk) = disk else {
|
||||
aligned.push(None);
|
||||
continue;
|
||||
};
|
||||
let index = reservations
|
||||
.iter()
|
||||
.position(|candidate| candidate.as_ref().is_some_and(|candidate| Arc::ptr_eq(&candidate.disk, disk)))?;
|
||||
aligned.push(reservations[index].take());
|
||||
}
|
||||
// Disks lost while staging need no activation; dropping their still-inert
|
||||
// reservations schedules cancellation without blocking the commit.
|
||||
Some(aligned)
|
||||
}
|
||||
|
||||
async fn activate_rename_metadata_reservations(
|
||||
mut reservations: Vec<Option<RenameMetadataReservation>>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
ctx: &Arc<crate::runtime::instance::InstanceContext>,
|
||||
) -> disk::error::Result<Option<Vec<Option<RenameMetadataReservation>>>> {
|
||||
// Tokens expire after 60 seconds. Before sending any activation, fall
|
||||
// back if a long upload consumed the safety margin for its RPC deadline.
|
||||
if reservations
|
||||
.iter()
|
||||
.flatten()
|
||||
.any(|reservation| reservation.prepared_at.elapsed() >= Duration::from_secs(50))
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
let activated = join_all(reservations.iter().flatten().map(|reservation| async move {
|
||||
if reservation.disk.is_local() {
|
||||
crate::store::admit_local_disk(ctx, &reservation.disk, None, false).await?;
|
||||
}
|
||||
let token = reservation.token.ok_or(DiskError::Unexpected)?;
|
||||
reservation
|
||||
.disk
|
||||
.activate_object_metadata(bucket, object, token, Duration::from_secs(5))
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
if activated.into_iter().any(|result| result.is_err()) {
|
||||
let cancelled = join_all(reservations.iter_mut().flatten().map(RenameMetadataReservation::cancel)).await;
|
||||
if cancelled.into_iter().any(|result| result.is_err()) {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(reservations))
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
@@ -3694,7 +3530,7 @@ type RenameDataLegacyTuple = (
|
||||
Option<OldCurrentSize>,
|
||||
);
|
||||
|
||||
pub(crate) fn put_rename_early_ack_enabled() -> bool {
|
||||
fn put_rename_early_ack_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, true)
|
||||
}
|
||||
|
||||
@@ -4060,7 +3896,6 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4157,8 +3992,6 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority: _,
|
||||
multipart_commit_owner: _,
|
||||
reservation_ctx,
|
||||
prepared_reservations,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -4174,11 +4007,6 @@ impl SetDisks {
|
||||
|
||||
let disk_count = disks.len();
|
||||
let fanout_disks = disks.to_vec();
|
||||
let mut reservations = match (reservation_ctx.as_ref(), prepared_reservations) {
|
||||
(Some(ctx), Some(prepared)) => activate_rename_metadata_reservations(prepared, dst_bucket, dst_object, ctx).await?,
|
||||
(_, _) => None,
|
||||
};
|
||||
let tail_has_metadata_reservations = reservations.is_some();
|
||||
let fanout_fence_tokens = Self::scanner_publication_lease_tokens_for_disks(disks, scanner_publication_lease_tokens)?;
|
||||
let coordinator_disks = fanout_disks.clone();
|
||||
let src_bucket = Arc::new(src_bucket.to_string());
|
||||
@@ -4211,11 +4039,6 @@ impl SetDisks {
|
||||
let file_info = file_info.clone();
|
||||
let successful_rename_completion_rank = successful_rename_completion_rank.clone();
|
||||
let namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let reservation = reservations
|
||||
.as_mut()
|
||||
.and_then(|entries| entries.get_mut(i))
|
||||
.and_then(Option::take);
|
||||
let reservation_ctx = reservation_ctx.clone();
|
||||
tasks.spawn(async move {
|
||||
let _namespace_commit_guard = namespace_commit_guard.clone();
|
||||
let mut dispatch_state = RenameDispatchState::NotDispatched;
|
||||
@@ -4226,12 +4049,6 @@ impl SetDisks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
let disk_admission = if reservation.is_some() && disk.is_local() {
|
||||
let ctx = reservation_ctx.as_ref().ok_or(DiskError::Unexpected)?;
|
||||
crate::store::admit_local_disk(ctx, &disk, None, true).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let disk_wait_location_stage = if disk.is_local() {
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT_LOCAL
|
||||
} else {
|
||||
@@ -4257,11 +4074,6 @@ impl SetDisks {
|
||||
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
dispatch_state = RenameDispatchState::MayHavePublished;
|
||||
let _foreground_active_guard = rename_foreground_active_guard(i);
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> = if disk_admission.is_some() {
|
||||
Some(Arc::new((namespace_commit_guard.clone(), disk_admission)))
|
||||
} else {
|
||||
namespace_commit_guard.clone().map(|owner| owner as Arc<dyn Send + Sync>)
|
||||
};
|
||||
let observed = disk
|
||||
.rename_data_borrowed_with_fence_observed(
|
||||
&src_bucket,
|
||||
@@ -4271,8 +4083,9 @@ impl SetDisks {
|
||||
&dst_object,
|
||||
disk::RenameDataGuards {
|
||||
scanner_publication_lease_token,
|
||||
metadata_reservation_token: reservation.as_ref().and_then(|owner| owner.token),
|
||||
namespace_owner,
|
||||
namespace_owner: namespace_commit_guard
|
||||
.clone()
|
||||
.map(|owner| owner as Arc<dyn Send + Sync>),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
@@ -4550,7 +4363,6 @@ impl SetDisks {
|
||||
let attach_tail_owner_started = rustfs_io_metrics::put_stage_timer();
|
||||
commit.map(|mut commit| {
|
||||
commit.tail_drain = Some(tail_drain);
|
||||
commit.tail_has_metadata_reservations = tail_has_metadata_reservations;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_ATTACH_TAIL_OWNER,
|
||||
attach_tail_owner_started,
|
||||
@@ -4620,8 +4432,6 @@ impl SetDisks {
|
||||
rollback_receipt,
|
||||
multipart_proof_authority,
|
||||
multipart_commit_owner,
|
||||
reservation_ctx: _,
|
||||
prepared_reservations: _,
|
||||
} = fence_options;
|
||||
if let Some(file_info) = disks
|
||||
.iter()
|
||||
@@ -5091,7 +4901,6 @@ impl SetDisks {
|
||||
old_current_source,
|
||||
committed_file_info,
|
||||
tail_drain: None,
|
||||
tail_has_metadata_reservations: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -5531,6 +5340,14 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(dst_object, true))
|
||||
})
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
let transaction_path = part_transaction_path(dst_object);
|
||||
let transaction_meta_path = format!("{transaction_path}/{PART_TRANSACTION_NEW_META}");
|
||||
|
||||
@@ -5730,6 +5547,14 @@ impl SetDisks {
|
||||
write_quorum: usize,
|
||||
) -> disk::error::Result<()> {
|
||||
let disks = self.get_disks_internal().await;
|
||||
if disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref()
|
||||
.is_some_and(|disk| disk.has_clean_active_multipart_source(part_path, false))
|
||||
})
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let listings = join_all(disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
async move {
|
||||
@@ -5937,7 +5762,7 @@ impl SetDisks {
|
||||
let all_prepared_succeeded = errs
|
||||
.iter()
|
||||
.zip(prepared_disks.iter())
|
||||
.all(|(err, disk)| disk.is_none() || err.is_none());
|
||||
.all(|(err, disk)| err.is_none() && disk.as_ref().is_some_and(|d| d.is_local()));
|
||||
let committed = if all_prepared_succeeded {
|
||||
let settles = prepared_disks.iter().map(|disk| {
|
||||
let disk = disk.clone();
|
||||
|
||||
@@ -161,7 +161,7 @@ impl SetDisks {
|
||||
)
|
||||
});
|
||||
let mut metadata = FileMeta::new();
|
||||
for parts in versions.into_values() {
|
||||
for (version_id, parts) in versions {
|
||||
let errors = parts
|
||||
.iter()
|
||||
.zip(&missing)
|
||||
@@ -173,19 +173,36 @@ impl SetDisks {
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
|
||||
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((read, write)) => (
|
||||
usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
|
||||
usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
|
||||
),
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
|
||||
Err(DiskError::ErasureReadQuorum) if complete => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
|
||||
let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
|
||||
// A null-version system record must carry the same commit proof
|
||||
// as GET. Listing only needs the selected metadata, so no disk
|
||||
// handles are retained from the shared read selector.
|
||||
Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
|
||||
} else {
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
|
||||
};
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
return Ok(None);
|
||||
// With no decodable version, the per-version quorum loop cannot
|
||||
// reject corrupt or unavailable replicas. Only complete, readable
|
||||
// evidence may establish that this key has no authoritative version.
|
||||
return if complete {
|
||||
Ok(None)
|
||||
} else {
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
};
|
||||
}
|
||||
Ok(Some(MetaCacheEntry {
|
||||
name,
|
||||
|
||||
@@ -357,8 +357,7 @@ use crate::io_support::rio::HashReader;
|
||||
|
||||
pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024;
|
||||
pub const MAX_PARTS_COUNT: usize = 10000;
|
||||
pub(crate) const RUSTFS_MULTIPART_BUCKET_KEY: &str = "x-rustfs-internal-multipart-bucket";
|
||||
pub(crate) const RUSTFS_MULTIPART_OBJECT_KEY: &str = "x-rustfs-internal-multipart-object";
|
||||
pub(crate) use crate::disk::multipart_index::{RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY};
|
||||
pub(crate) const DATA_MOVEMENT_MULTIPART_PREFIX: &str = "data-movement";
|
||||
const ENV_ISSUE3031_DIAG_ENABLE: &str = "RUSTFS_ISSUE3031_DIAG_ENABLE";
|
||||
|
||||
@@ -410,10 +409,6 @@ impl ObjectLockDiagGuard {
|
||||
self
|
||||
}
|
||||
|
||||
async fn downgrade_committed_tail(&mut self) -> bool {
|
||||
self.guard.downgrade_committed_tail().await
|
||||
}
|
||||
|
||||
/// Whether the underlying namespace lock's heartbeat has observed a
|
||||
/// refresh-quorum loss (backlog#899 Phase 2). Callers fence their commit
|
||||
/// point on this so a stale lock holder does not race a double-write.
|
||||
@@ -906,10 +901,6 @@ mod core;
|
||||
pub(crate) use core::io_primitives::disk_call_counters;
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, rename_fanout_barrier};
|
||||
pub(crate) use core::io_primitives::{
|
||||
RenameMetadataReservation, align_rename_metadata_reservations, prepare_rename_metadata_reservations,
|
||||
put_rename_early_ack_enabled,
|
||||
};
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
@@ -4704,6 +4695,17 @@ pub(crate) fn same_distributed_lock_domain(left: &[Arc<dyn LockClient>], right:
|
||||
.all(|right_client| left.iter().any(|left_client| Arc::ptr_eq(left_client, right_client)))
|
||||
}
|
||||
|
||||
// Reusing a held writer additionally requires the same quorum, so repeated
|
||||
// clients must not give different weight to an otherwise identical domain.
|
||||
fn same_distributed_lock_quorum(left: &[Arc<dyn LockClient>], right: &[Arc<dyn LockClient>]) -> bool {
|
||||
!left.is_empty()
|
||||
&& left.len() == right.len()
|
||||
&& left.iter().all(|client| {
|
||||
left.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
&& right.iter().filter(|other| Arc::ptr_eq(client, other)).count() == 1
|
||||
})
|
||||
}
|
||||
|
||||
const ERASURE_CACHE_MAX_ENTRIES: usize = 32;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
@@ -5016,6 +5018,11 @@ impl DiskHealthEntry {
|
||||
#[error("committed system metadata write requires repair by its owning writer")]
|
||||
pub(crate) struct SystemMetadataRepairRequired;
|
||||
|
||||
/// An inline metadata write was rejected before staging or commit began.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("system metadata write rejected before staging: {0}")]
|
||||
pub(crate) struct SystemMetadataWriteNotStarted(#[source] pub(crate) Error);
|
||||
|
||||
impl SetDisks {
|
||||
pub(in crate::set_disk) async fn persist_delete_marker_purge(
|
||||
&self,
|
||||
@@ -5653,9 +5660,9 @@ impl SetDisks {
|
||||
pub(crate) async fn shares_namespace_lock_domain(&self, other: &Self) -> bool {
|
||||
match (self.ctx.is_dist_erasure().await, other.ctx.is_dist_erasure().await) {
|
||||
(false, false) => Arc::ptr_eq(&self.local_lock_manager, &other.local_lock_manager),
|
||||
(true, true) => {
|
||||
self.set_lock_namespace == other.set_lock_namespace && same_distributed_lock_domain(&self.lockers, &other.lockers)
|
||||
}
|
||||
// Distributed lock clients receive the raw ObjectKey; the pool
|
||||
// namespace is not part of the key stored by their backends.
|
||||
(true, true) => same_distributed_lock_quorum(&self.lockers, &other.lockers),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -9046,6 +9053,13 @@ mod tests {
|
||||
"constructing an object lock must not clone each client Arc"
|
||||
);
|
||||
|
||||
// Check shared ownership before acquisition can spawn pending cleanup.
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
|
||||
let write_error = write_lock
|
||||
.get_write_lock(Duration::from_millis(500))
|
||||
.await
|
||||
@@ -9060,11 +9074,6 @@ mod tests {
|
||||
),
|
||||
"the shared client representation must preserve the exact write quorum result: {write_error}"
|
||||
);
|
||||
let read_lock = set
|
||||
.new_ns_lock("bucket", "read-object")
|
||||
.await
|
||||
.expect("second namespace lock should be created");
|
||||
assert_eq!(Arc::strong_count(&set.shared_lockers), clients_before + 2);
|
||||
let read_guard = read_lock
|
||||
.get_read_lock(Duration::from_millis(500))
|
||||
.await
|
||||
@@ -9888,7 +9897,7 @@ mod tests {
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn namespace_lock_domain_includes_pool_namespace() {
|
||||
async fn distributed_namespace_lock_domain_follows_backend_not_pool_name() {
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
|
||||
let first: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
@@ -9927,11 +9936,25 @@ mod tests {
|
||||
"sets in the same pool share the object namespace lock domain"
|
||||
);
|
||||
assert!(
|
||||
!same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pool namespaces must not be deduplicated solely by identical clients"
|
||||
same_pool_first_set.shares_namespace_lock_domain(&other_pool_set).await,
|
||||
"different pools using identical backends lock the same raw object key"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distributed_lock_domain_rejects_partial_or_duplicate_backends() {
|
||||
let a: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let b: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let c: Arc<dyn LockClient> = Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
assert!(same_distributed_lock_quorum(&[a.clone(), b.clone()], &[b.clone(), a.clone()]));
|
||||
assert!(!same_distributed_lock_quorum(&[a.clone(), b.clone()], &[a.clone(), c]));
|
||||
assert!(!same_distributed_lock_quorum(
|
||||
&[a.clone(), a.clone(), b.clone()],
|
||||
&[a.clone(), b.clone(), b]
|
||||
));
|
||||
assert!(!same_distributed_lock_quorum(&[], &[]));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn streaming_reader_holds_read_lock_until_eof() {
|
||||
|
||||
@@ -141,6 +141,7 @@ pub(crate) struct DeleteRecoverySummary {
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DeleteRecoveryContext<'a> {
|
||||
pub incarnation: Option<Uuid>,
|
||||
pub work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
pub dry_run: bool,
|
||||
pub explicit_delete: bool,
|
||||
/// Only the locked heal path may repair a live marker-creation source.
|
||||
@@ -156,6 +157,7 @@ impl<'a> DeleteRecoveryContext<'a> {
|
||||
pub(crate) fn new(incarnation: Option<Uuid>, can_commit: &'a (dyn Fn() -> bool + Sync)) -> Self {
|
||||
Self {
|
||||
incarnation,
|
||||
work_budget: None,
|
||||
dry_run: false,
|
||||
explicit_delete: false,
|
||||
repair_marker_source: false,
|
||||
@@ -230,19 +232,54 @@ async fn inspect(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
budget: Option<&crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
) -> Vec<Result<DeleteRecoveryInspection>> {
|
||||
join_all(disks.iter().map(|disk| async move {
|
||||
let state = disk
|
||||
.as_ref()
|
||||
.ok_or(DiskError::DiskNotFound)?
|
||||
.inspect_delete_transaction(bucket, object, transaction)
|
||||
.await?;
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
let state = if let Some(budget) = budget {
|
||||
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
|
||||
let reply = disk
|
||||
.inspect_delete_transaction_bounded(bucket, object, transaction, admitted)
|
||||
.await?;
|
||||
if reply.visited > admitted {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.charge(reply.visited);
|
||||
reply.inspection
|
||||
} else {
|
||||
disk.inspect_delete_transaction(bucket, object, transaction).await?
|
||||
};
|
||||
validate(&state, object, transaction)?;
|
||||
Ok(state)
|
||||
}))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn discover_bounded(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
budget: &crate::object_api::DeleteRecoveryWorkBudget,
|
||||
) -> Result<Vec<DeleteTransactionId>> {
|
||||
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
|
||||
let page = disk.discover_delete_transactions_bounded(bucket, object, admitted).await?;
|
||||
if page.visited > admitted
|
||||
|| page.transactions.len() > 256
|
||||
|| page
|
||||
.transactions
|
||||
.iter()
|
||||
.any(|id| id.object != object || id.transaction.is_nil())
|
||||
|| (!page.complete && !page.transactions.is_empty())
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.charge(page.visited);
|
||||
if !page.complete {
|
||||
return Err(DiskError::DeleteRecoveryWorkLimit);
|
||||
}
|
||||
Ok(page.transactions)
|
||||
}
|
||||
|
||||
async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Result<Vec<DeleteTransactionId>> {
|
||||
let mut cursor = None;
|
||||
let mut ids = BTreeSet::new();
|
||||
@@ -273,30 +310,92 @@ async fn discover(disk: &DiskStore, bucket: &str, object: Option<&str>) -> Resul
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct RecoveryWorkScope<'a> {
|
||||
namespace_owner: Option<&'a Arc<dyn Send + Sync>>,
|
||||
budget: Option<&'a crate::object_api::DeleteRecoveryWorkBudget>,
|
||||
}
|
||||
|
||||
async fn settle_outcome(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
state: &DeleteRecoveryInspection,
|
||||
action: DeleteRecoveryAction,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
) -> Result<crate::disk::DeleteRecoverySettlementOutcome> {
|
||||
use crate::disk::DeleteRecoverySettlementOutcome;
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let outcome = if let Some(budget) = scope.budget {
|
||||
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
|
||||
let may_publish = matches!(&action, DeleteRecoveryAction::PublishCommit | DeleteRecoveryAction::AdoptCommit { .. });
|
||||
let reply = disk
|
||||
.settle_delete_transaction_bounded(
|
||||
bucket,
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: state.id.object.clone(),
|
||||
transaction: state.id.transaction,
|
||||
expected_fingerprint: state.fingerprint,
|
||||
action,
|
||||
max_entries: admitted,
|
||||
},
|
||||
scope.namespace_owner.cloned(),
|
||||
)
|
||||
.await?;
|
||||
if reply.visited > admitted
|
||||
|| (!may_publish && matches!(&reply.outcome, DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred))
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.charge(reply.visited);
|
||||
reply.outcome
|
||||
} else {
|
||||
DeleteRecoverySettlementOutcome::Inspected(Box::new(
|
||||
disk.settle_delete_transaction_with_namespace_owner(
|
||||
bucket,
|
||||
&state.id.object,
|
||||
state.id.transaction,
|
||||
state.fingerprint,
|
||||
action,
|
||||
scope.namespace_owner.cloned(),
|
||||
)
|
||||
.await?,
|
||||
))
|
||||
};
|
||||
if let DeleteRecoverySettlementOutcome::Inspected(next) = &outcome {
|
||||
validate(next, &state.id.object, state.id.transaction)?;
|
||||
}
|
||||
Ok(outcome)
|
||||
}
|
||||
|
||||
async fn settle(
|
||||
disk: &DiskStore,
|
||||
bucket: &str,
|
||||
state: &DeleteRecoveryInspection,
|
||||
action: DeleteRecoveryAction,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
namespace_owner: Option<&Arc<dyn Send + Sync>>,
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
match settle_outcome(disk, bucket, state, action, can_commit, scope).await? {
|
||||
crate::disk::DeleteRecoverySettlementOutcome::Inspected(inspection) => Ok(*inspection),
|
||||
crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred => {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn has_receipt_quorum(outcomes: &[Result<crate::disk::DeleteRecoverySettlementOutcome>], quorum: usize) -> bool {
|
||||
outcomes.iter().filter(|outcome| published_receipt(outcome)).count() >= quorum
|
||||
}
|
||||
|
||||
fn published_receipt(outcome: &Result<crate::disk::DeleteRecoverySettlementOutcome>) -> bool {
|
||||
match outcome {
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => receipt(state.state),
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => true,
|
||||
Err(_) => false,
|
||||
}
|
||||
let result = disk
|
||||
.settle_delete_transaction_with_namespace_owner(
|
||||
bucket,
|
||||
&state.id.object,
|
||||
state.id.transaction,
|
||||
state.fingerprint,
|
||||
action,
|
||||
namespace_owner.cloned(),
|
||||
)
|
||||
.await;
|
||||
let next = result?;
|
||||
validate(&next, &state.id.object, state.id.transaction)?;
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
/// Each phase fans out across independent drives, and awaits all started work
|
||||
@@ -307,23 +406,16 @@ async fn settle_group(
|
||||
copies: &mut [DeleteRecoveryInspection],
|
||||
action: impl Fn(&DeleteRecoveryInspection) -> Option<DeleteRecoveryAction> + Sync,
|
||||
can_commit: &(impl Fn() -> bool + Sync + ?Sized),
|
||||
namespace_owner: Option<&Arc<dyn Send + Sync>>,
|
||||
scope: RecoveryWorkScope<'_>,
|
||||
removed_versions: &mut HashSet<Uuid>,
|
||||
) -> Result<bool> {
|
||||
let results = join_all(disks.iter().zip(copies.iter()).map(|(disk, copy)| {
|
||||
let action = action(copy);
|
||||
async move {
|
||||
match action {
|
||||
Some(action) => settle(
|
||||
disk.as_ref().ok_or(DiskError::DiskNotFound)?,
|
||||
bucket,
|
||||
copy,
|
||||
action,
|
||||
can_commit,
|
||||
namespace_owner,
|
||||
)
|
||||
.await
|
||||
.map(Some),
|
||||
Some(action) => settle(disk.as_ref().ok_or(DiskError::DiskNotFound)?, bucket, copy, action, can_commit, scope)
|
||||
.await
|
||||
.map(Some),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
@@ -439,16 +531,31 @@ impl SetDisks {
|
||||
true
|
||||
}
|
||||
};
|
||||
let mut fast_receipt_quorum = false;
|
||||
if can_try_local_fast_path && can_commit() {
|
||||
let fast_published = join_all(disks.iter().map(|disk| async {
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
disk.try_publish_prepared_delete_local(bucket, object, transaction, context.namespace_owner.clone())
|
||||
.await
|
||||
disk.try_publish_prepared_delete_local(
|
||||
bucket,
|
||||
object,
|
||||
transaction,
|
||||
context.namespace_owner.clone(),
|
||||
context.work_budget.clone(),
|
||||
)
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
fast_receipt_quorum = fast_published.iter().filter(|result| matches!(result, Ok(Some(_)))).count() >= write_quorum;
|
||||
if fast_receipt_quorum
|
||||
&& fast_published
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
let all_fast_published = fast_published.len() == self.set_drive_count
|
||||
&& fast_published.iter().all(|res| matches!(res, Ok(Some((_, true, true)))));
|
||||
if all_fast_published && can_commit() {
|
||||
@@ -456,6 +563,7 @@ impl SetDisks {
|
||||
test_hooks::record_cleanup_attempt(bucket, object);
|
||||
let finalized = join_all(disks.iter().zip(fast_published.iter()).map(|(disk, pub_res)| {
|
||||
let ns_owner = context.namespace_owner.clone();
|
||||
let work_budget = context.work_budget.clone();
|
||||
async move {
|
||||
if !can_commit() {
|
||||
return Ok(false);
|
||||
@@ -466,11 +574,24 @@ impl SetDisks {
|
||||
let Ok(Some((journal, _, _))) = pub_res else {
|
||||
return Ok(false);
|
||||
};
|
||||
disk.try_finish_and_finalize_prepared_delete_local(bucket, object, transaction, journal, ns_owner)
|
||||
.await
|
||||
disk.try_finish_and_finalize_prepared_delete_local(
|
||||
bucket,
|
||||
object,
|
||||
transaction,
|
||||
journal,
|
||||
ns_owner,
|
||||
work_budget,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
if finalized
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if finalized.iter().all(|res| matches!(res, Ok(true))) {
|
||||
if let Some(ticket) = context.recovery_ticket.as_ref() {
|
||||
self.ctx.delete_recovery_hints().ack(ticket);
|
||||
@@ -479,7 +600,17 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
}
|
||||
let states = inspect(disks, bucket, object, transaction).await;
|
||||
let states = inspect(disks, bucket, object, transaction, context.work_budget.as_deref()).await;
|
||||
if states
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return if fast_receipt_quorum {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
};
|
||||
}
|
||||
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
|
||||
if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum
|
||||
&& states.iter().all(|state| {
|
||||
@@ -499,25 +630,50 @@ impl SetDisks {
|
||||
if state.state == DeleteRecoveryState::LegacyAmbiguous {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
settle(
|
||||
settle_outcome(
|
||||
disk,
|
||||
bucket,
|
||||
state,
|
||||
DeleteRecoveryAction::PublishCommit,
|
||||
can_commit,
|
||||
context.namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: context.namespace_owner.as_ref(),
|
||||
budget: context.work_budget.as_deref(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
if published
|
||||
.iter()
|
||||
.filter(|state| matches!(state, Ok(state) if receipt(state.state)))
|
||||
.count()
|
||||
< write_quorum
|
||||
{
|
||||
return Err(DiskError::ErasureWriteQuorum);
|
||||
if !has_receipt_quorum(&published, write_quorum) {
|
||||
return Err(
|
||||
if published
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
DiskError::DeleteRecoveryWorkLimit
|
||||
} else {
|
||||
DiskError::ErasureWriteQuorum
|
||||
},
|
||||
);
|
||||
}
|
||||
if published.iter().any(|outcome| {
|
||||
matches!(
|
||||
outcome,
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
|
||||
)
|
||||
}) {
|
||||
return Ok(());
|
||||
}
|
||||
let published = published
|
||||
.into_iter()
|
||||
.map(|outcome| match outcome {
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) => Ok(*state),
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred) => {
|
||||
Err(DiskError::DeleteRecoveryWorkLimit)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
// Physical cleanup is retryable after the certified decision. An
|
||||
// unavailable participant retains the decision for its eventual return.
|
||||
let ticket = context.recovery_ticket.clone();
|
||||
@@ -670,6 +826,7 @@ impl SetDisks {
|
||||
}
|
||||
let DeleteRecoveryContext {
|
||||
incarnation,
|
||||
work_budget,
|
||||
dry_run,
|
||||
explicit_delete,
|
||||
repair_marker_source,
|
||||
@@ -681,7 +838,13 @@ impl SetDisks {
|
||||
} = context;
|
||||
let disks = self.disk_inventory().await;
|
||||
let complete_inventory = disks.len() == self.set_drive_count && disks.iter().all(Option::is_some);
|
||||
let pages = join_all(disks.iter().flatten().map(|disk| discover(disk, bucket, Some(object)))).await;
|
||||
let pages = join_all(disks.iter().flatten().map(|disk| async {
|
||||
match work_budget.as_deref() {
|
||||
Some(budget) => discover_bounded(disk, bucket, object, budget).await,
|
||||
None => discover(disk, bucket, Some(object)).await,
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
let mut transactions = BTreeSet::new();
|
||||
let mut inspected = 0;
|
||||
let mut offline = !complete_inventory;
|
||||
@@ -712,6 +875,7 @@ impl SetDisks {
|
||||
..Default::default()
|
||||
};
|
||||
let mut all_states = BTreeMap::new();
|
||||
let mut registered_evidence_hint = false;
|
||||
for transaction in transactions {
|
||||
let mut complete = Vec::with_capacity(disks.len());
|
||||
let inspections = if let Some((published_tx, published_states)) = published_hint
|
||||
@@ -723,8 +887,14 @@ impl SetDisks {
|
||||
{
|
||||
published_states.to_vec()
|
||||
} else {
|
||||
inspect(&disks, bucket, object, transaction).await
|
||||
inspect(&disks, bucket, object, transaction, work_budget.as_deref()).await
|
||||
};
|
||||
if inspections
|
||||
.iter()
|
||||
.any(|result| matches!(result, Err(DiskError::DeleteRecoveryWorkLimit)))
|
||||
{
|
||||
return Err(DiskError::DeleteRecoveryWorkLimit);
|
||||
}
|
||||
for state in inspections {
|
||||
match state {
|
||||
Ok(state) => {
|
||||
@@ -736,6 +906,33 @@ impl SetDisks {
|
||||
summary.blocked += 1;
|
||||
return Ok(summary);
|
||||
}
|
||||
// Candidate UUID directories are not evidence. Only a bound, inspected
|
||||
// journal can create a foreground hint; empty/partial scans rely on the
|
||||
// existing raw anti-entropy obligation instead of queuing arbitrary prefixes.
|
||||
if work_budget.is_some()
|
||||
&& !registered_evidence_hint
|
||||
&& journal.protocol_version == DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
&& journal.volume == bucket
|
||||
&& journal.object == object
|
||||
&& journal.transaction == transaction
|
||||
&& matches!(
|
||||
state.state,
|
||||
DeleteRecoveryState::NewPrepared
|
||||
| DeleteRecoveryState::QuorumCommitted
|
||||
| DeleteRecoveryState::CommitFinished
|
||||
)
|
||||
{
|
||||
self.ctx.delete_recovery_hints().register(
|
||||
crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: self.pool_index,
|
||||
set: self.set_index,
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
incarnation,
|
||||
},
|
||||
);
|
||||
registered_evidence_hint = true;
|
||||
}
|
||||
}
|
||||
complete.push(state);
|
||||
}
|
||||
@@ -767,7 +964,7 @@ impl SetDisks {
|
||||
};
|
||||
if let Some((transaction, source)) = prerequisite {
|
||||
Box::pin(self.heal_delete_recovery_source(bucket, object, &source, can_commit, namespace_owner.as_ref())).await?;
|
||||
let refreshed = inspect(&disks, bucket, object, transaction)
|
||||
let refreshed = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -853,7 +1050,7 @@ impl SetDisks {
|
||||
if mutated {
|
||||
// A sibling may have restored metadata or reclaimed shared
|
||||
// evidence. Refresh fingerprints before acting on this group.
|
||||
*copies = inspect(&disks, bucket, object, transaction)
|
||||
*copies = inspect(&disks, bucket, object, transaction, work_budget.as_deref())
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -876,7 +1073,10 @@ impl SetDisks {
|
||||
(copy.state == DeleteRecoveryState::LegacyAmbiguous).then_some(DeleteRecoveryAction::DiscardLegacy)
|
||||
},
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -987,7 +1187,10 @@ impl SetDisks {
|
||||
)
|
||||
},
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?;
|
||||
@@ -1001,7 +1204,10 @@ impl SetDisks {
|
||||
.then_some(DeleteRecoveryAction::FinishCommit)
|
||||
},
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?;
|
||||
@@ -1048,7 +1254,10 @@ impl SetDisks {
|
||||
copies,
|
||||
|copy| (copy.state == DeleteRecoveryState::NewPrepared).then_some(DeleteRecoveryAction::Rollback),
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1076,7 +1285,7 @@ impl SetDisks {
|
||||
settled.push(copies);
|
||||
} else {
|
||||
for transaction in all_states.keys() {
|
||||
let fresh = inspect(&disks, bucket, object, *transaction).await;
|
||||
let fresh = inspect(&disks, bucket, object, *transaction, work_budget.as_deref()).await;
|
||||
if fresh.iter().any(|state| !matches!(state, Ok(state) if matches!(state.state, DeleteRecoveryState::Absent | DeleteRecoveryState::CommitFinished))) {
|
||||
summary.pending += 1;
|
||||
return Ok(summary);
|
||||
@@ -1086,7 +1295,7 @@ impl SetDisks {
|
||||
}
|
||||
for (index, mut copies) in settled.into_iter().enumerate() {
|
||||
if index > 0 {
|
||||
copies = inspect(&disks, bucket, object, copies[0].id.transaction)
|
||||
copies = inspect(&disks, bucket, object, copies[0].id.transaction, work_budget.as_deref())
|
||||
.await
|
||||
.into_iter()
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
@@ -1106,7 +1315,10 @@ impl SetDisks {
|
||||
&mut copies,
|
||||
|copy| (copy.state == DeleteRecoveryState::CommitFinished).then_some(DeleteRecoveryAction::FinalizeCommit),
|
||||
can_commit,
|
||||
namespace_owner.as_ref(),
|
||||
RecoveryWorkScope {
|
||||
namespace_owner: namespace_owner.as_ref(),
|
||||
budget: work_budget.as_deref(),
|
||||
},
|
||||
&mut summary.removed_versions,
|
||||
)
|
||||
.await?
|
||||
@@ -1358,6 +1570,459 @@ mod tests {
|
||||
body
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_mixed_inspected_and_deferred_receipts_count_only_real_quorum() {
|
||||
let bucket = "recover-bounded-mixed-quorum";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![73; 4096], false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let mut outcomes = Vec::new();
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let state = disk
|
||||
.inspect_delete_transaction_bounded(bucket, "object", transaction, 65536)
|
||||
.await
|
||||
.expect("preparation inspection");
|
||||
let max_entries = match index {
|
||||
0 => 65536,
|
||||
1 | 2 => state.visited,
|
||||
_ => 1,
|
||||
};
|
||||
outcomes.push(
|
||||
disk.settle_delete_transaction_bounded(
|
||||
bucket,
|
||||
crate::disk::BoundedDeleteRecoverySettlement {
|
||||
object: "object".to_owned(),
|
||||
transaction,
|
||||
expected_fingerprint: state.inspection.fingerprint,
|
||||
action: DeleteRecoveryAction::PublishCommit,
|
||||
max_entries,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.map(|reply| reply.outcome),
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
matches!(&outcomes[0], Ok(crate::disk::DeleteRecoverySettlementOutcome::Inspected(state)) if receipt(state.state))
|
||||
);
|
||||
assert!(matches!(
|
||||
&outcomes[1],
|
||||
Ok(crate::disk::DeleteRecoverySettlementOutcome::ReceiptPublishedInspectionDeferred)
|
||||
));
|
||||
assert!(matches!(outcomes[3], Err(crate::disk::error::DiskError::DeleteRecoveryWorkLimit)));
|
||||
assert!(has_receipt_quorum(&outcomes, 3));
|
||||
assert!(!has_receipt_quorum(&outcomes, 4));
|
||||
outcomes[2] = Err(DiskError::FileAccessDenied);
|
||||
assert!(
|
||||
!has_receipt_quorum(&outcomes, 3),
|
||||
"a write failure cannot substitute for a durable receipt"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_single_and_batch_share_bounded_work_across_versions_and_retries() {
|
||||
use crate::object_api::DeleteRecoveryWorkBudget;
|
||||
use crate::storage_api_contracts::object::ObjectToDelete;
|
||||
let bucket = "recover-bounded-request";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
for dir in &dirs {
|
||||
let prefix = dir.path().join(bucket).join("prefix");
|
||||
std::fs::create_dir_all(&prefix).expect("physical prefix");
|
||||
for _ in 0..64 {
|
||||
std::fs::create_dir(prefix.join(Uuid::new_v4().to_string())).expect("legitimate child");
|
||||
}
|
||||
}
|
||||
let error = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"prefix",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(DeleteRecoveryWorkBudget::new(8))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("incomplete discovery is not absence");
|
||||
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
|
||||
for incarnation in [
|
||||
None,
|
||||
Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation"),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "prefix".to_owned(),
|
||||
incarnation,
|
||||
}),
|
||||
"an unproven prefix must not move its scan into background recovery"
|
||||
);
|
||||
}
|
||||
for distinct_versions in [false, true] {
|
||||
let budget = Arc::new(DeleteRecoveryWorkBudget::new(32_768));
|
||||
let objects = (0..1000)
|
||||
.map(|_| ObjectToDelete {
|
||||
object_name: "prefix".to_owned(),
|
||||
version_id: distinct_versions.then(Uuid::new_v4),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
objects,
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::clone(&budget)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors
|
||||
.iter()
|
||||
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
|
||||
"repeated actions must not each obtain a new allowance: {distinct_versions}"
|
||||
);
|
||||
assert!(dirs.iter().all(|dir| {
|
||||
std::fs::read_dir(dir.path().join(bucket).join("prefix"))
|
||||
.map(|entries| entries.count() == 64)
|
||||
.unwrap_or(false)
|
||||
}));
|
||||
}
|
||||
let error = store
|
||||
.delete_object(bucket, "prefix", ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("ordinary missing key");
|
||||
assert!(
|
||||
crate::error::is_err_object_not_found(&error),
|
||||
"small complete inventory retains no-op semantics: {error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
let mut dirs = Vec::new();
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
for set_index in 0..2 {
|
||||
for disk_index in 0..4 {
|
||||
let dir = tempfile::tempdir().expect("temporary disk");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("endpoint");
|
||||
endpoint.set_pool_index(pool_index);
|
||||
endpoint.set_set_index(set_index);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoints.push(endpoint);
|
||||
dirs.push(dir);
|
||||
}
|
||||
}
|
||||
pools.push(PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 2,
|
||||
drives_per_set: 4,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: format!("bounded-recovery-{pool_index}"),
|
||||
platform: "test".to_owned(),
|
||||
});
|
||||
}
|
||||
let endpoints = EndpointServerPools(pools);
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
crate::store::init_local_disks_with_instance_ctx(&ctx, endpoints.clone())
|
||||
.await
|
||||
.expect("local disks");
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let store = ECStore::new_with_instance_ctx("127.0.0.1:0".parse().expect("address"), endpoints, cancel.clone(), ctx)
|
||||
.await
|
||||
.expect("two pools with two sets");
|
||||
cancel.cancel();
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "recover-bounded-fanout";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket");
|
||||
let candidates = (0..100).map(|index| format!("prefix-{index}")).collect::<Vec<_>>();
|
||||
let mut keys = BTreeSet::new();
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
keys.insert(
|
||||
candidates
|
||||
.iter()
|
||||
.find(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set))
|
||||
.expect("fixture key for set")
|
||||
.clone(),
|
||||
);
|
||||
}
|
||||
}
|
||||
let keys = keys.into_iter().collect::<Vec<_>>();
|
||||
for pool in &store.pools {
|
||||
for set in &pool.disk_set {
|
||||
assert!(
|
||||
keys.iter().any(|key| Arc::ptr_eq(&pool.get_disks_by_key(key), set)),
|
||||
"fixture must cover every set"
|
||||
);
|
||||
}
|
||||
}
|
||||
for dir in &dirs {
|
||||
for key in &keys {
|
||||
let prefix = dir.path().join(bucket).join(key);
|
||||
std::fs::create_dir_all(&prefix).expect("physical prefix");
|
||||
for _ in 0..8 {
|
||||
std::fs::create_dir(prefix.join(Uuid::new_v4().to_string())).expect("child");
|
||||
}
|
||||
}
|
||||
}
|
||||
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(32_768));
|
||||
let objects = (0..1000)
|
||||
.map(|index| crate::storage_api_contracts::object::ObjectToDelete {
|
||||
object_name: keys[index % keys.len()].clone(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
..Default::default()
|
||||
})
|
||||
.collect();
|
||||
let (_, errors) = store
|
||||
.delete_objects(
|
||||
bucket,
|
||||
objects,
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::clone(&budget)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
errors
|
||||
.iter()
|
||||
.any(|error| matches!(error, Some(StorageError::DeleteRecoveryWorkLimit))),
|
||||
"all pools and sets must share the finite allowance"
|
||||
);
|
||||
assert!(budget.remaining() < 32_768, "fanout must debit the original request allowance");
|
||||
assert!(dirs.iter().all(|dir| {
|
||||
std::fs::read_dir(dir.path().join(bucket).join(&keys[0]))
|
||||
.map(|entries| entries.count() == 8)
|
||||
.unwrap_or(false)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_fast_commit_keeps_success_when_cleanup_budget_is_empty() {
|
||||
let bucket = "recover-bounded-fast-commit";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![74; 2 * 1024 * 1024], false).await;
|
||||
// Inline cleanup needs no directory enumeration and may legitimately
|
||||
// finish with zero scan credits. Require real part files here.
|
||||
for disk in &disks {
|
||||
let info = disk
|
||||
.read_version("", bucket, "object", "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read non-inline source");
|
||||
assert!(!info.inline_data(), "the fixture must require physical part cleanup");
|
||||
let data_dir = info.data_dir.expect("physical data directory");
|
||||
let part = std::path::PathBuf::from(disk.endpoint().to_string())
|
||||
.join(bucket)
|
||||
.join("object")
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1");
|
||||
assert!(part.is_file(), "the cleanup scan must encounter a real part");
|
||||
}
|
||||
store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(0))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a cached prepared receipt commits independently of cleanup budget");
|
||||
assert!(
|
||||
store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "object".to_owned(),
|
||||
incarnation: Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation")
|
||||
),
|
||||
})
|
||||
);
|
||||
assert!(
|
||||
!disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 128)
|
||||
.await
|
||||
.expect("retained recovery proof")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("background finishes committed cleanup");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_unverified_uuid_candidate_does_not_enqueue_background_work() {
|
||||
let bucket = "recover-bounded-candidate";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
let transaction = Uuid::new_v4();
|
||||
for directory in &dirs {
|
||||
let candidate = directory.path().join(bucket).join("candidate").join(transaction.to_string());
|
||||
std::fs::create_dir_all(&candidate).expect("candidate path");
|
||||
std::fs::write(candidate.join("delete-prepared"), b"not a bound journal").expect("unverified candidate");
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.delete_object(bucket, "candidate", ObjectOptions::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for incarnation in [
|
||||
None,
|
||||
Some(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation"),
|
||||
),
|
||||
] {
|
||||
assert!(
|
||||
!store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.is_pending(&crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "candidate".to_owned(),
|
||||
incarnation,
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_budget_refusal_preserves_evidence_for_background_replay() {
|
||||
let bucket = "recover-bounded-evidence";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
let payload = vec![71; 4096];
|
||||
put(&store, bucket, &payload, false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let before = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("before refusal");
|
||||
let error = store
|
||||
.delete_object(
|
||||
bucket,
|
||||
"object",
|
||||
ObjectOptions {
|
||||
delete_recovery_work_budget: Some(Arc::new(crate::object_api::DeleteRecoveryWorkBudget::new(1))),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("no incomplete recovery admission");
|
||||
assert!(matches!(error, StorageError::DeleteRecoveryWorkLimit), "{error:?}");
|
||||
let after = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("after refusal");
|
||||
assert_eq!(before.fingerprint, after.fingerprint);
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("background-compatible recovery");
|
||||
assert_eq!(body(&store, bucket, None).await, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_committed_delete_defers_exhausted_cleanup_without_rollback() {
|
||||
let bucket = "recover-bounded-commit";
|
||||
let (_dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![72; 4096], false).await;
|
||||
let transaction = applied(&store, &disks, bucket, None, disks.len()).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let key = crate::store::delete_recovery_runtime::DeleteRecoveryKey {
|
||||
pool: 0,
|
||||
set: 0,
|
||||
bucket: bucket.to_owned(),
|
||||
object: "object".to_owned(),
|
||||
incarnation: None,
|
||||
};
|
||||
let ticket = set.ctx.delete_recovery_hints().register(key.clone());
|
||||
let budget = Arc::new(crate::object_api::DeleteRecoveryWorkBudget::default());
|
||||
let exhausted_after_commit = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
// This hook disables local fast cleanup and runs only after receipt
|
||||
// publication and the shared recovery path's key-wide GC inspection.
|
||||
// Exhausting preparation instead would correctly reject the delete.
|
||||
test_hooks::before_gc_finalize(bucket, "object", {
|
||||
let budget = Arc::clone(&budget);
|
||||
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
|
||||
Arc::new(move || {
|
||||
let budget = Arc::clone(&budget);
|
||||
let exhausted_after_commit = Arc::clone(&exhausted_after_commit);
|
||||
Box::pin(async move {
|
||||
assert!(budget.reserve(u32::MAX).expect("credits remain until committed cleanup") > 0);
|
||||
exhausted_after_commit.store(true, std::sync::atomic::Ordering::Release);
|
||||
})
|
||||
})
|
||||
});
|
||||
set.publish_delete_commit(
|
||||
bucket,
|
||||
"object",
|
||||
transaction,
|
||||
&set.disk_inventory().await,
|
||||
set.set_drive_count / 2 + 1,
|
||||
DeleteRecoveryContext {
|
||||
work_budget: Some(Arc::clone(&budget)),
|
||||
recovery_ticket: ticket,
|
||||
..DeleteRecoveryContext::new(None, &|| true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("the durable decision still succeeds");
|
||||
assert!(exhausted_after_commit.load(std::sync::atomic::Ordering::Acquire));
|
||||
assert_eq!(budget.remaining(), 0);
|
||||
assert!(set.ctx.delete_recovery_hints().is_pending(&key));
|
||||
let state = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("retained decision");
|
||||
assert!(receipt(state.state));
|
||||
store
|
||||
.recover_delete_transactions_at_incarnation(bucket, Some("object"), None, &HealOpts::default())
|
||||
.await
|
||||
.expect("eventual cleanup");
|
||||
assert!(
|
||||
disks[0]
|
||||
.discover_delete_transactions(bucket, Some("object"), None, 128)
|
||||
.await
|
||||
.expect("cleaned proofs")
|
||||
.transactions
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_set_rolls_back_undecided_inline_bytes() {
|
||||
let bucket = "recover-undecided";
|
||||
|
||||
@@ -34,7 +34,6 @@ mod heal_lock_wait_tests;
|
||||
#[derive(Clone, Copy)]
|
||||
struct ExplicitVersionHeal<'a> {
|
||||
opts: &'a HealOpts,
|
||||
staging_guard: Option<&'a Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>>,
|
||||
allow_regeneration: bool,
|
||||
preserve_unrepairable: bool,
|
||||
commit_fence: Option<&'a (dyn Fn() -> bool + Sync)>,
|
||||
@@ -42,58 +41,6 @@ struct ExplicitVersionHeal<'a> {
|
||||
retirement: Option<&'a MarkerRetirementContext<'a>>,
|
||||
}
|
||||
|
||||
async fn restore_heal_exclusive_guard(
|
||||
guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>,
|
||||
) -> disk::error::Result<()> {
|
||||
tokio::time::timeout(get_lock_acquire_timeout(), async {
|
||||
loop {
|
||||
{
|
||||
let mut guard = guard.lock().await;
|
||||
if guard.is_lock_lost() {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
if tokio::time::timeout(std::time::Duration::from_secs(1), guard.try_upgrade_committed_tail())
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
// A writer may have waited on a peer that upgraded while
|
||||
// another peer was still waiting for its readers to drain.
|
||||
let waiting = tokio::time::timeout(std::time::Duration::from_secs(1), guard.waiting_writer()).await;
|
||||
return if matches!(waiting, Ok(Some(false))) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(DiskError::Timeout)
|
||||
};
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| DiskError::Timeout)?
|
||||
}
|
||||
|
||||
async fn wait_for_heal_writer(guard: &Arc<tokio::sync::Mutex<rustfs_lock::NamespaceLockGuard>>) {
|
||||
loop {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
let waiting =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(1), async { guard.lock().await.waiting_writer().await }).await;
|
||||
if !matches!(waiting, Ok(Some(false))) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_heal_staging_without_waiting(set: SetDisks, tmp_id: String) {
|
||||
drop(tokio::spawn(async move {
|
||||
let _ = set.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}));
|
||||
}
|
||||
|
||||
// Keep the large heal state machine and its construction temporary out of
|
||||
// each dispatch caller's poll frame. Ordinary object writes do not use this
|
||||
// heal-only allocation boundary.
|
||||
@@ -897,7 +844,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: None,
|
||||
@@ -1048,7 +994,6 @@ impl SetDisks {
|
||||
) -> disk::error::Result<(HealResultItem, Option<DiskError>)> {
|
||||
let ExplicitVersionHeal {
|
||||
opts,
|
||||
staging_guard,
|
||||
allow_regeneration: allow_explicit_version_regen,
|
||||
preserve_unrepairable,
|
||||
commit_fence,
|
||||
@@ -1756,29 +1701,7 @@ impl SetDisks {
|
||||
let dst_data_dir = data_dir;
|
||||
|
||||
if !latest_meta.deleted && !latest_meta.is_remote() {
|
||||
let mut shared_staging = false;
|
||||
if let Some(guard) = staging_guard {
|
||||
let supported = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.supports_heal_lock_transition().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if supported {
|
||||
let converted = tokio::time::timeout(std::time::Duration::from_secs(1), async {
|
||||
guard.lock().await.downgrade_committed_tail().await
|
||||
})
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if converted {
|
||||
shared_staging = true;
|
||||
} else {
|
||||
// Some peers may have converted. Abort without committing;
|
||||
// dropping the guard releases each retained lease.
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
let stage = async {
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = async {
|
||||
let erasure_info = latest_meta.erasure.clone();
|
||||
let mut writer_failures = HealWriterFailures::new(disks_to_heal_count);
|
||||
|
||||
@@ -2003,43 +1926,17 @@ impl SetDisks {
|
||||
|
||||
writer_failures.warn(bucket, object, version_id, "partial_targets_unavailable");
|
||||
Ok(None)
|
||||
};
|
||||
let stage_result: disk::error::Result<Option<DiskError>> = if shared_staging {
|
||||
if let Some(guard) = staging_guard {
|
||||
tokio::select! {
|
||||
result = stage => result,
|
||||
_ = wait_for_heal_writer(guard) => {
|
||||
// The UUID prefix is private to this attempt. Cleanup must not
|
||||
// retain the namespace lock while a foreground writer waits;
|
||||
// the stale-tmp sweeper also covers a cancelled disk operation.
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(DiskError::Timeout);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
}
|
||||
} else {
|
||||
stage.await
|
||||
};
|
||||
}
|
||||
.await;
|
||||
match stage_result {
|
||||
Ok(None) => {}
|
||||
failure => {
|
||||
if shared_staging {
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
} else {
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
}
|
||||
// Release readers and writers before removing all parts
|
||||
// staged under this heal's exclusively owned prefix.
|
||||
let _ = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_id).await;
|
||||
return failure.map(|error| (result, error));
|
||||
}
|
||||
}
|
||||
if shared_staging
|
||||
&& let Some(guard) = staging_guard
|
||||
&& let Err(error) = restore_heal_exclusive_guard(guard).await
|
||||
{
|
||||
cleanup_heal_staging_without_waiting(self.clone(), tmp_id.clone());
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
let _read_repair_write_lock_guard = if let Some(expected_fingerprint) = read_repair_commit_fingerprint {
|
||||
if _read_repair_read_lock_guard
|
||||
@@ -3719,7 +3616,6 @@ impl SetDisks {
|
||||
&version,
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(can_commit),
|
||||
@@ -3780,7 +3676,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &read_opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: false,
|
||||
preserve_unrepairable: true,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -3807,7 +3702,7 @@ impl SetDisks {
|
||||
.new_ns_lock(bucket, object)
|
||||
.await
|
||||
.map_err(|e| e.narrow_to_disk().unwrap_or_else(DiskError::other))?;
|
||||
Some(Arc::new(tokio::sync::Mutex::new(
|
||||
Some(Arc::new(
|
||||
measure_heal_object_lock(
|
||||
"heal_object",
|
||||
HealObjectLockKind::Write,
|
||||
@@ -3819,7 +3714,7 @@ impl SetDisks {
|
||||
.narrow_to_disk()
|
||||
.unwrap_or_else(DiskError::other)
|
||||
})?,
|
||||
)))
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -3837,18 +3732,10 @@ impl SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let write_lock_loss = if let Some(guard) = &_write_lock_guard {
|
||||
guard.lock().await.lock_lost_signal()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let locks_lost = || {
|
||||
write_lock_loss.as_ref().is_some_and(|signal| signal.is_lost())
|
||||
_write_lock_guard.as_deref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| transaction_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
};
|
||||
// Recovery cleanup can outlive its future. Never permit reads until
|
||||
// every physical operation holding its recovery owner has finished.
|
||||
let recovery_tail = Arc::new(());
|
||||
let mut recovery_removed_version = false;
|
||||
if !opts.read_repair
|
||||
&& _write_lock_guard.is_some()
|
||||
@@ -3875,7 +3762,6 @@ impl SetDisks {
|
||||
Arc::new((
|
||||
scope.clone(),
|
||||
self.ctx.begin_namespace_commit(),
|
||||
Arc::clone(&recovery_tail),
|
||||
_write_lock_guard.clone(),
|
||||
transaction_guard.clone(),
|
||||
)) as Arc<dyn Send + Sync>
|
||||
@@ -3999,9 +3885,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -4026,9 +3909,6 @@ impl SetDisks {
|
||||
version_id,
|
||||
ExplicitVersionHeal {
|
||||
opts: &inner_opts,
|
||||
staging_guard: (Arc::strong_count(&recovery_tail) == 1)
|
||||
.then_some(_write_lock_guard.as_ref())
|
||||
.flatten(),
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&commit_fence),
|
||||
@@ -6167,7 +6047,6 @@ mod heal_result_report_tests {
|
||||
"",
|
||||
ExplicitVersionHeal {
|
||||
opts: &opts,
|
||||
staging_guard: None,
|
||||
allow_regeneration: true,
|
||||
preserve_unrepairable: false,
|
||||
commit_fence: Some(&can_commit),
|
||||
@@ -10758,369 +10637,6 @@ mod heal_result_report_tests {
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_storage_entry_keeps_readers_available_during_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-storage-entry";
|
||||
let object = "object.bin";
|
||||
let expected = vec![0x37; 1024 * 1024];
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected.clone()),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external object data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(&missing).await.expect("remove shard");
|
||||
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
let result = heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
read_repair: false,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
(result, absence)
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("read-repair reached shard reconstruction");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_millis(200),
|
||||
set.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("background heal reconstruction should allow concurrent readers")
|
||||
.expect("open object while reconstruction is paused");
|
||||
let mut during_heal = Vec::new();
|
||||
tokio::time::timeout(Duration::from_secs(10), reader.stream.read_to_end(&mut during_heal))
|
||||
.await
|
||||
.expect("read completes while reconstruction is paused")
|
||||
.expect("read object during reconstruction");
|
||||
assert_eq!(during_heal, expected);
|
||||
drop(reader);
|
||||
release.notify_one();
|
||||
let (result, absence) = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("repair finishes")
|
||||
.expect("repair task joins");
|
||||
let (_, error) = result.expect("repair operation succeeds");
|
||||
assert!(error.is_none(), "repair failed: {error:?}");
|
||||
assert!(absence.is_none());
|
||||
assert!(missing.exists(), "missing shard was not restored");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read repaired object");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_does_not_timeout_foreground_put_during_long_reconstruction() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-long-put";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original object");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("read metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
let mut absence = None;
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let put = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x52; 1024]), &ObjectOptions::default()),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes after releasing reconstruction")
|
||||
.expect("heal task joins");
|
||||
put.expect("foreground PUT finishes during reconstruction")
|
||||
.expect("foreground PUT must not time out");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
drop(_pause);
|
||||
let mut absence = None;
|
||||
let (_, retry_error) = set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut absence,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a later heal must retry the current object");
|
||||
assert!(retry_error.is_none(), "retry must complete: {retry_error:?}");
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read foreground write after heal");
|
||||
let mut actual = Vec::new();
|
||||
reader.stream.read_to_end(&mut actual).await.expect("read full object");
|
||||
assert_eq!(actual, vec![0x52; 1024], "stale heal must not overwrite the foreground write");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn background_heal_yields_to_version_delete_without_resurrecting_it() {
|
||||
use crate::object_api::WriteCompletion;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let (dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let bucket = "background-heal-version-delete";
|
||||
let object = "object.bin";
|
||||
set.make_bucket(
|
||||
bucket,
|
||||
&MakeBucketOptions {
|
||||
versioning_enabled: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create versioned bucket");
|
||||
let original = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x37; 1024 * 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write original version");
|
||||
let version = original.version_id.expect("version id").to_string();
|
||||
let survivor = set
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x52; 1024]),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write surviving version")
|
||||
.version_id
|
||||
.expect("surviving version id");
|
||||
let metadata = disks[0]
|
||||
.read_version("", bucket, object, &version, &ReadOptions::default())
|
||||
.await
|
||||
.expect("read original metadata");
|
||||
let missing = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join(object)
|
||||
.join(metadata.data_dir.expect("external data").to_string())
|
||||
.join("part.1");
|
||||
tokio::fs::remove_file(missing).await.expect("remove shard");
|
||||
let (_pause, arrived, release) =
|
||||
ReadRepairPauseScope::install_at(bucket, object, super::ReadRepairPausePhase::PartStaging(1));
|
||||
let heal_set = set.clone();
|
||||
let heal_version = version.clone();
|
||||
let heal = tokio::spawn(async move {
|
||||
heal_set
|
||||
.heal_object_with_retirement(
|
||||
bucket,
|
||||
object,
|
||||
&heal_version,
|
||||
&HealOpts {
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
&mut None,
|
||||
AbsenceProofRequest {
|
||||
retirement: None,
|
||||
allow_unversioned: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), arrived.notified())
|
||||
.await
|
||||
.expect("reconstruction started");
|
||||
let delete = tokio::time::timeout(
|
||||
Duration::from_secs(7),
|
||||
set.delete_object(
|
||||
bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
object_lock_config_snapshot: Some(Arc::new(crate::set_disk::ObjectLockConfigSnapshot::new(
|
||||
crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
|
||||
))),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
)
|
||||
.await;
|
||||
release.notify_one();
|
||||
let heal_result = tokio::time::timeout(Duration::from_secs(20), heal)
|
||||
.await
|
||||
.expect("heal finishes")
|
||||
.expect("heal task joins");
|
||||
delete
|
||||
.expect("foreground delete finishes")
|
||||
.expect("foreground delete succeeds");
|
||||
assert!(
|
||||
matches!(heal_result, Err(Error::Timeout)),
|
||||
"interrupted heal must be retryable: {heal_result:?}"
|
||||
);
|
||||
let result = set
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(version.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&result, Err(Error::VersionNotFound(b, o, v)) if b == bucket && o == object && v == &version),
|
||||
"stale heal must not resurrect the deleted version: {result:?}"
|
||||
);
|
||||
let mut survivor = set
|
||||
.get_object_reader(
|
||||
bucket,
|
||||
object,
|
||||
None,
|
||||
http::HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(survivor.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("surviving version remains readable");
|
||||
let mut body = Vec::new();
|
||||
survivor.stream.read_to_end(&mut body).await.expect("read surviving version");
|
||||
assert_eq!(body, vec![0x52; 1024]);
|
||||
for disk in &disks {
|
||||
let physical = disk.read_version("", bucket, object, &version, &ReadOptions::default()).await;
|
||||
assert!(
|
||||
matches!(physical, Err(DiskError::FileNotFound | DiskError::FileVersionNotFound)),
|
||||
"deleted version must remain absent on each disk: {physical:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_repair_storage_entry_does_not_prove_missing_versions_absent() {
|
||||
let (_dirs, _disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
|
||||
@@ -95,13 +95,8 @@ impl SetDisks {
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> Result<Arc<rustfs_lock::NamespaceLockGuard>> {
|
||||
let resource = ObjectKey::with_version(
|
||||
bucket,
|
||||
object_name_disk_identity(object).as_ref(),
|
||||
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
|
||||
);
|
||||
let guard = self
|
||||
.namespace_lock_for_key(resource)
|
||||
.namespace_lock_for_key(self.delete_transaction_resource(bucket, object))
|
||||
.await
|
||||
.get_write_lock(super::super::get_lock_acquire_timeout())
|
||||
.await
|
||||
@@ -111,6 +106,94 @@ impl SetDisks {
|
||||
Ok(Arc::new(guard))
|
||||
}
|
||||
|
||||
fn delete_transaction_resource(&self, bucket: &str, object: &str) -> ObjectKey {
|
||||
ObjectKey::with_version(
|
||||
bucket,
|
||||
object_name_disk_identity(object).as_ref(),
|
||||
format!("rustfs:delete-transaction:{}:{}", self.pool_index, self.set_index),
|
||||
)
|
||||
}
|
||||
|
||||
/// Batched write-lock acquisition for the whole key set of one bulk
|
||||
/// mutation: one lock round instead of one quorum round-trip per key.
|
||||
///
|
||||
/// Bulk delete is the only multi-key write-lock holder. The batched round
|
||||
/// is deadlock-free by construction (a single `lock_batch` RPC per lock
|
||||
/// client, and every lock server orders concurrent batches by the same
|
||||
/// global (shard, key) order), so unlike an optimistic probe wave it
|
||||
/// needs no canonical serial fallback. Outcomes are per key and use the
|
||||
/// same error shape as the single-key paths.
|
||||
async fn namespace_write_locks_batch(
|
||||
&self,
|
||||
bucket: &str,
|
||||
resources: &[ObjectKey],
|
||||
timeout: std::time::Duration,
|
||||
mode: &'static str,
|
||||
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
|
||||
if resources.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let lock = self.namespace_lock_for_key(resources[0].clone()).await;
|
||||
lock.get_write_locks_batch(resources, timeout)
|
||||
.await
|
||||
.into_iter()
|
||||
.zip(resources)
|
||||
.map(|(result, resource)| {
|
||||
result.map_err(|error| self.map_namespace_lock_error(bucket, resource.object.as_ref(), mode, error))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Batched raw namespace write locks for a bulk delete's key set. The
|
||||
/// resource keys use the same disk-identity folding as `new_ns_lock`, so
|
||||
/// aliases (`a\b` vs `a/b`) still share one lock.
|
||||
pub(crate) async fn acquire_object_write_locks_batch(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: &[&str],
|
||||
timeout: std::time::Duration,
|
||||
) -> Vec<Result<rustfs_lock::NamespaceLockGuard>> {
|
||||
let resources: Vec<ObjectKey> = objects
|
||||
.iter()
|
||||
.map(|object| ObjectKey {
|
||||
bucket: Arc::from(bucket),
|
||||
object: Arc::from(object_name_disk_identity(object).as_ref()),
|
||||
version: None,
|
||||
})
|
||||
.collect();
|
||||
self.namespace_write_locks_batch(bucket, &resources, timeout, "write").await
|
||||
}
|
||||
|
||||
/// Batched delete-transaction locks for a bulk delete's key set; see
|
||||
/// [`Self::acquire_delete_transaction_lock`] for the resource shape.
|
||||
pub(in crate::set_disk) async fn acquire_delete_transaction_locks_batch(
|
||||
&self,
|
||||
bucket: &str,
|
||||
objects: &[&str],
|
||||
timeout: std::time::Duration,
|
||||
) -> Vec<Result<Arc<rustfs_lock::NamespaceLockGuard>>> {
|
||||
let resources: Vec<ObjectKey> = objects
|
||||
.iter()
|
||||
.map(|object| self.delete_transaction_resource(bucket, object))
|
||||
.collect();
|
||||
let mut guards = Vec::with_capacity(resources.len());
|
||||
for (object, result) in objects.iter().zip(
|
||||
self.namespace_write_locks_batch(bucket, &resources, timeout, "delete_transaction")
|
||||
.await,
|
||||
) {
|
||||
match result {
|
||||
Ok(guard) => {
|
||||
#[cfg(test)]
|
||||
super::delete_recovery::test_hooks::run_after_transaction_lock_acquired(bucket, object, &guard);
|
||||
let _ = object;
|
||||
guards.push(Ok(Arc::new(guard)));
|
||||
}
|
||||
Err(error) => guards.push(Err(error)),
|
||||
}
|
||||
}
|
||||
guards
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "lock diagnostics formatter with no caller in this port (backlog#1823)")]
|
||||
pub(in crate::set_disk) fn format_lock_error(&self, bucket: &str, object: &str, mode: &str, err: &LockResult) -> String {
|
||||
match err {
|
||||
@@ -375,7 +458,10 @@ impl SetDisks {
|
||||
return None;
|
||||
}
|
||||
|
||||
let _ = disk.close().await;
|
||||
// A failed remote reconnect must retain its existing recovery monitor.
|
||||
if disk.is_local() {
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
}
|
||||
|
||||
let endpoint = self.set_endpoints.get(i)?;
|
||||
@@ -497,19 +583,26 @@ impl SetDisks {
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
|
||||
if !new_disk.is_local() {
|
||||
let mut disks = self.disks.write().await;
|
||||
let slot = &mut disks[disk_idx];
|
||||
let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
|
||||
_ => false,
|
||||
let displaced = {
|
||||
let mut disks = self.disks.write().await;
|
||||
let slot = &mut disks[disk_idx];
|
||||
let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
|
||||
_ => false,
|
||||
};
|
||||
// A GET probe or another reconnect may have filled this slot while
|
||||
// format I/O was pending. Only the successful publisher owns the
|
||||
// displaced monitor; a failed or superseded probe must retain it.
|
||||
if unchanged {
|
||||
new_disk.enable_health_check();
|
||||
slot.replace(new_disk)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
// A GET probe or another reconnect may have filled this slot while
|
||||
// format I/O was pending. Do not replace its handle or start a
|
||||
// second health monitor for an unpublished handle.
|
||||
if unchanged {
|
||||
new_disk.enable_health_check();
|
||||
*slot = Some(new_disk);
|
||||
if let Some(disk) = displaced {
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user