mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-28 15:02:49 +00:00
Compare commits
34 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2b243d8659 | |||
| c52570d385 | |||
| e0b1635abf | |||
| f0f2b0cf48 | |||
| 0882e99c32 | |||
| 802ac8a1bd | |||
| 9408b90f13 | |||
| 08ac1ec218 | |||
| 388d1f17e6 | |||
| 22c093e66b | |||
| 24ab34a5ee | |||
| 9068d83400 | |||
| 20fc897036 | |||
| a86117590c | |||
| ceb53fef85 | |||
| 4e0ff037ec | |||
| fa0ac13e9b | |||
| c7848df9a0 | |||
| 4502a1f9b0 | |||
| 93d51d172f | |||
| b80590f1a7 | |||
| 025ed4c6d1 | |||
| 3b428bffef | |||
| da52303f2a | |||
| a79c6072f1 | |||
| ab512768d5 | |||
| adf17bbfba | |||
| fb5c002a30 | |||
| 8a94c64934 | |||
| 9ca0e9b717 | |||
| bd83bd3f6b | |||
| 34803e8165 | |||
| 9af2f7c04e | |||
| 9d6f48c325 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=b8937d6de585895a837b1505c7a9f91aa45394fbdc47cd179287a35a6f1de8be
|
||||
sha256-linux=e803ee5fbd29bd45b8542b4256b75d5a2196ccc7ccfd1ccac6e14553730e9047
|
||||
sha256-darwin=1520cd3027c3a6924afc6a204eddd5475bc30f0109d4c2847456a66177263244
|
||||
sha256-linux=26155b7f1c5bf6bf596ff24a4136e6767ea4ee6461c32db7fbe1400791da2c28
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=ce7b419faf1ff79840dfc6325d31ee8d1ca5eab85125b94d0bcd58cdf07f0f76
|
||||
sha256-linux=5e9e3d701ee873c13fb9094575f6889779c6078b1107624bb57663bef9b9d21f
|
||||
sha256-darwin=347a0d4e5e547d5388703062ce66a66167a845b6e57ec591ccb4c732a567c68d
|
||||
sha256-linux=4f9efa47194279185bb68249e82c788fa147abe944b6c764050e3d47f31e7926
|
||||
|
||||
@@ -52,6 +52,8 @@ script-tests: ## Run shell script tests
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
|
||||
|
||||
@@ -88,6 +88,11 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests.
|
||||
[[profile.default.scripts]]
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
@@ -339,6 +344,11 @@ threads-required = "num-test-threads"
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(batch_transitioned_delete_uses_free_version_per_item|decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|dispatched_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|force_tier_remove_blocks_on_physical_free_version_hidden_by_other_pool|legacy_unknown_transition_delete_falls_back_for_single_batch_and_blocks_prefix|multi_pool_(recursive_prefix_rejects_legacy_or_hidden_merge_loser_before_delete|same_remote_tuple_(batch|single)_delete_waits_for_all_sources|same_tuple_recursive_prefix_uses_one_journal_owner|transitioned_delete_persists_one_free_version_per_remote_tuple)|recursive_prefix_partial_(pool|set)_failure_keeps_prepared_cleanup_owners|restored_transitioned_delete_uses_free_version_as_cleanup_owner|stable_transitioned_recursive_prefix_delete_uses_journal_owners|suspended_null_transition_delete_uses_free_version_as_sole_owner|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)|transitioned_delete_(free_version_replays_after_store_restart|local_quorum_failure_rolls_back_without_cleanup_owner|uses_free_version_as_cleanup_owner)|versioned_delete_marker_keeps_transitioned_source_and_remote_object|versioned_explicit_transition_delete_preserves_other_version_then_allows_bucket_delete))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
# These cross-crate real-storage fixtures need the same bounded stack as ECStore tests.
|
||||
[[profile.ci.scripts]]
|
||||
filter = '(package(rustfs) & test(/^(admin::handlers::site_replication::tests::durable_iam_state_apply_iam_item_rejects_a_stale_recreate_after_a_replicated_delete|admin::service::federated_identity::tests::(binding_keeps_explicit_iam_mapping_strict|markerless_legacy_oidc_session_crosses_auth_and_iam_deletion_boundaries)|storage::scanner_dirty_journal::tests::(clear_callback_keeps_unknown_coverage_until_all_pending_work_is_confirmed|owned_journals_keep_peer_records_after_interleaved_writes_and_clear))$/)) | (package(rustfs-heal) & ((binary(heal_b5_versioned_regression_test) & test(/^serial_tests::(test_heal_unversioned_bucket_e2e|test_historical_null_recursive_heal_restores_physical_versions|test_recursive_deep_heal_reports_stale_delete_marker_repair)$/)) | (binary(usage_observation_read_repair_test) & test(=usage_observation_read_repair_survives_snapshot_cleanup)))) | (package(rustfs-test-utils) & test(=data_usage_snapshot_tests::data_usage_snapshot_recovery_wires_primary_backup_and_negative_cache))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
@@ -1748,7 +1748,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_objects_count{job=~\"$job\", bucket=~\"$bucket\"})",
|
||||
"expr": "max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"})",
|
||||
"legendFormat": "{{bucket}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -1848,7 +1848,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (job, bucket) (rustfs_cluster_usage_buckets_total_bytes{job=~\"$job\", bucket=~\"$bucket\"})",
|
||||
"expr": "max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id=\"$storage_cluster\", job=~\"$job\", bucket=~\"$bucket\"})",
|
||||
"legendFormat": "{{bucket}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
|
||||
@@ -61,6 +61,7 @@ scrape_configs:
|
||||
otlp:
|
||||
promote_resource_attributes:
|
||||
- service.instance.id
|
||||
- process.pid
|
||||
- service.name
|
||||
- service.namespace
|
||||
- cloud.availability_zone
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
evaluation_interval: 1m
|
||||
tests:
|
||||
- name: bucket usage keeps clusters separate and deduplicates reporting nodes
|
||||
interval: 1m
|
||||
input_series:
|
||||
- series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a1"}
|
||||
values: '4096 1024'
|
||||
- series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a2"}
|
||||
values: '4096 1024'
|
||||
- series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="b",job="rustfs",bucket="shared",service_instance_id="b1"}
|
||||
values: '65536 65536'
|
||||
- series: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job="rustfs",bucket="empty",service_instance_id="a1"}
|
||||
values: '0 0'
|
||||
- series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a1"}
|
||||
values: '3 1'
|
||||
- series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="shared",service_instance_id="a2"}
|
||||
values: '3 1'
|
||||
- series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="b",job="rustfs",bucket="shared",service_instance_id="b1"}
|
||||
values: '70 70'
|
||||
- series: rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job="rustfs",bucket="empty",service_instance_id="a1"}
|
||||
values: '0 0'
|
||||
promql_expr_test:
|
||||
- expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",job=~"rustfs",bucket=~".*"})
|
||||
eval_time: 1m
|
||||
exp_samples:
|
||||
- labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}'
|
||||
value: 1024
|
||||
- labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}'
|
||||
value: 0
|
||||
- expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_objects_count{rustfs_cluster_id="a",job=~"rustfs",bucket=~".*"})
|
||||
eval_time: 1m
|
||||
exp_samples:
|
||||
- labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}'
|
||||
value: 1
|
||||
- labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}'
|
||||
value: 0
|
||||
- expr: max by (rustfs_cluster_id, job, bucket) (rustfs_cluster_usage_buckets_total_bytes)
|
||||
eval_time: 1m
|
||||
exp_samples:
|
||||
- labels: '{rustfs_cluster_id="a",job="rustfs",bucket="shared"}'
|
||||
value: 1024
|
||||
- labels: '{rustfs_cluster_id="a",job="rustfs",bucket="empty"}'
|
||||
value: 0
|
||||
- labels: '{rustfs_cluster_id="b",job="rustfs",bucket="shared"}'
|
||||
value: 65536
|
||||
- expr: rustfs_cluster_usage_buckets_total_bytes{rustfs_cluster_id="a",bucket="unknown"}
|
||||
eval_time: 1m
|
||||
exp_samples: []
|
||||
@@ -0,0 +1,39 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Set up Docker Compose
|
||||
description: Install the verified Compose CLI used by native Linux x86_64 Connect acceptance
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install verified Docker Compose CLI plugin
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ $(uname -s) == Linux && $(uname -m) == x86_64 ]]
|
||||
plugin_dir="${DOCKER_CONFIG:-$HOME/.docker}/cli-plugins"
|
||||
mkdir -p "$plugin_dir"
|
||||
compose_download=$(mktemp "${RUNNER_TEMP:?}/connect-compose.XXXXXX")
|
||||
compose_staged=$(mktemp "$plugin_dir/.docker-compose.XXXXXX")
|
||||
trap 'rm -f "$compose_download" "$compose_staged"' EXIT
|
||||
curl --fail --location --retry 3 \
|
||||
https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-linux-x86_64 \
|
||||
--output "$compose_download"
|
||||
printf '%s %s\n' \
|
||||
dba9d98e1ba5bfe11d88c99b9bd32fc4a0624a30fafe68eea34d61a3e42fd372 \
|
||||
"$compose_download" | sha256sum --check --strict
|
||||
install -m 0755 "$compose_download" "$compose_staged"
|
||||
mv -f "$compose_staged" "$plugin_dir/docker-compose"
|
||||
[[ $(docker compose version --short) == 2.40.3 ]]
|
||||
@@ -36,6 +36,16 @@ inputs:
|
||||
description: "Install cross-compilation tools"
|
||||
required: false
|
||||
default: "false"
|
||||
zig-version:
|
||||
description: >-
|
||||
Zig version for cargo-zigbuild builds. Pinned so the `<target>.<glibc>`
|
||||
baseline semantics cannot drift with an untested zig release.
|
||||
required: false
|
||||
default: "0.17.0"
|
||||
cargo-zigbuild-version:
|
||||
description: "cargo-zigbuild version installed for cross builds."
|
||||
required: false
|
||||
default: "0.23.4"
|
||||
target:
|
||||
description: "Target architecture to add"
|
||||
required: false
|
||||
@@ -99,10 +109,14 @@ runs:
|
||||
- name: Install Zig
|
||||
if: inputs.install-cross-tools == 'true'
|
||||
uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2
|
||||
with:
|
||||
version: ${{ inputs.zig-version }}
|
||||
|
||||
- name: Install cargo-zigbuild
|
||||
if: inputs.install-cross-tools == 'true'
|
||||
uses: taiki-e/install-action@a21ae4029b089b9ddc45704028756f51ab8abe48 # cargo-zigbuild
|
||||
with:
|
||||
tool: cargo-zigbuild@${{ inputs.cargo-zigbuild-version }}
|
||||
|
||||
- name: Install cargo-nextest
|
||||
if: inputs.install-test-tools == 'true'
|
||||
@@ -126,5 +140,5 @@ runs:
|
||||
# normalises, so cargo fingerprints stay valid.
|
||||
cache-all-crates: false
|
||||
cache-on-failure: true
|
||||
shared-key: ${{ inputs.cache-shared-key }}
|
||||
shared-key: ${{ inputs.cache-shared-key }}${{ inputs.install-cross-tools == 'true' && format('-zig-{0}-zigbuild-{1}', inputs.zig-version, inputs.cargo-zigbuild-version) || '' }}
|
||||
save-if: ${{ inputs.cache-save-if }}
|
||||
|
||||
+56
-39
@@ -213,12 +213,12 @@ jobs:
|
||||
fi
|
||||
|
||||
all='{"include":[
|
||||
{"target_id":"linux-x86_64-musl","os":"sm-standard-4","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":""},
|
||||
{"target_id":"linux-aarch64-musl","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":""},
|
||||
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":"","features":"pyroscope"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-gnu","cross":false,"platform":"linux","rustflags":"","features":"pyroscope"},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope"},
|
||||
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":""}
|
||||
{"target_id":"linux-x86_64-musl","os":"sm-standard-4","target":"x86_64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-aarch64-musl","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"sm-standard-4-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
|
||||
]}'
|
||||
|
||||
if [[ "${selected}" == "all" ]]; then
|
||||
@@ -246,7 +246,8 @@ jobs:
|
||||
echo "Selected platforms: ${selected}"
|
||||
|
||||
build-rustfs:
|
||||
name: Build RustFS
|
||||
# Acceptance workflows select artifacts by this stable platform name.
|
||||
name: Build RustFS (${{ matrix.target_id }})
|
||||
needs: [ build-check, prepare-platform-matrix ]
|
||||
if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
|
||||
runs-on: ${{ matrix.os }}
|
||||
@@ -273,7 +274,7 @@ jobs:
|
||||
with:
|
||||
rust-version: stable
|
||||
target: ${{ matrix.target }}
|
||||
cache-shared-key: build-${{ matrix.target }}
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}
|
||||
# main only. A cache saved on refs/tags/X is scoped to that tag: no
|
||||
# other tag, no main run and no PR can restore it, so every release
|
||||
# cycle wrote up to 12 entries of 1-2GB (preview tag plus final tag,
|
||||
@@ -410,35 +411,53 @@ jobs:
|
||||
shell: bash
|
||||
env:
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_OS: ${{ matrix.platform }}-${{ startsWith(matrix.target, 'aarch64-') && 'aarch64' || 'x86_64' }}
|
||||
run: |
|
||||
# Force rebuild by touching build.rs
|
||||
touch rustfs/build.rs
|
||||
|
||||
BINARY_NAMES=(rustfs)
|
||||
if [[ "${{ matrix.platform }}" == "linux" ]]; then
|
||||
BINARY_NAMES+=(rustfs-cli)
|
||||
fi
|
||||
FEATURE_ARGS=()
|
||||
if [[ -n "${{ matrix.features }}" ]]; then
|
||||
FEATURE_ARGS=(--features "${{ matrix.features }}")
|
||||
fi
|
||||
|
||||
# Build one binary at a time so release LTO links cannot exhaust a
|
||||
# self-hosted runner by running concurrently.
|
||||
for binary in "${BINARY_NAMES[@]}"; do
|
||||
PROFILE_ARGS=()
|
||||
if [[ "$binary" == "rustfs" && "$BUILD_TARGET" == *-linux-gnu && "${{ matrix.features }}" == "pyroscope" ]]; then
|
||||
# The in-process profiler resolves names from the final executable.
|
||||
# Retain function symbols without enabling DWARF source information.
|
||||
PROFILE_ARGS=(--config 'profile.release.strip="none"')
|
||||
fi
|
||||
if [[ "${{ matrix.cross }}" == "true" ]]; then
|
||||
# All cross targets in the matrix are Linux; zigbuild handles them.
|
||||
cargo zigbuild --release --target ${{ matrix.target }} "${FEATURE_ARGS[@]}" "${PROFILE_ARGS[@]}" -p rustfs --bin "$binary"
|
||||
else
|
||||
cargo build --release --target ${{ matrix.target }} "${FEATURE_ARGS[@]}" "${PROFILE_ARGS[@]}" -p rustfs --bin "$binary"
|
||||
fi
|
||||
done
|
||||
# gnu targets pin an explicit glibc baseline (`<target>.<glibc>` for
|
||||
# cargo-zigbuild) so release binaries link no symbols newer than it
|
||||
# and run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
|
||||
TARGET_ARG="$BUILD_TARGET"
|
||||
if [[ -n "${{ matrix.glibc }}" ]]; then
|
||||
TARGET_ARG="${TARGET_ARG}.${{ matrix.glibc }}"
|
||||
fi
|
||||
|
||||
PROFILE_ARGS=()
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "${{ matrix.features }}" == "pyroscope" ]]; then
|
||||
# The in-process profiler resolves names from the final executable.
|
||||
# Retain function symbols without enabling DWARF source information.
|
||||
PROFILE_ARGS=(--config 'profile.release.strip="none"')
|
||||
fi
|
||||
if [[ "${{ matrix.cross }}" == "true" ]]; then
|
||||
cargo zigbuild --release --locked --target "$TARGET_ARG" "${FEATURE_ARGS[@]}" "${PROFILE_ARGS[@]}" -p rustfs --bin rustfs
|
||||
else
|
||||
cargo build --release --locked --target "$TARGET_ARG" "${FEATURE_ARGS[@]}" "${PROFILE_ARGS[@]}" -p rustfs --bin rustfs
|
||||
fi
|
||||
|
||||
# All release targets use same-architecture runners, including GNU
|
||||
# builds that use Zig only to select the glibc ABI floor.
|
||||
bin="target/${{ matrix.target }}/release/rustfs"
|
||||
if [[ -f "${bin}.exe" ]]; then
|
||||
bin="${bin}.exe"
|
||||
fi
|
||||
version_output=$("$bin" --version)
|
||||
if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
|
||||
! grep -Fxq "build os : ${BUILD_OS}" <<< "$version_output" || \
|
||||
! grep -Fxq "rust channel : stable-${BUILD_TARGET}" <<< "$version_output"; then
|
||||
echo "release binary reports an unexpected compile target" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "${{ matrix.glibc }}" ]]; then
|
||||
python3 scripts/release/check_glibc.py "${{ matrix.glibc }}" \
|
||||
"target/${{ matrix.target }}/release/rustfs"
|
||||
fi
|
||||
|
||||
- name: Verify macOS binary has no dynamic liblzma dependency
|
||||
if: matrix.platform == 'macos'
|
||||
@@ -554,9 +573,6 @@ jobs:
|
||||
fi
|
||||
|
||||
BINARY_FILES=("$BINARY_NAME")
|
||||
if [[ "${{ matrix.platform }}" == "linux" ]]; then
|
||||
BINARY_FILES+=("rustfs-cli")
|
||||
fi
|
||||
|
||||
if [[ "$TARGET" == *-linux-gnu && "${{ matrix.features }}" == "pyroscope" ]]; then
|
||||
BINARY_FILES+=(rustfs.cpu-symbol-catalog.json rustfs.cpu-symbol-catalog.sha256)
|
||||
@@ -671,7 +687,7 @@ jobs:
|
||||
echo "🔧 Build type: ${BUILD_TYPE}"
|
||||
echo "📊 Version: ${VERSION}"
|
||||
|
||||
- name: Verify packaged Linux binaries
|
||||
- name: Verify packaged Linux binary
|
||||
if: matrix.platform == 'linux'
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -679,7 +695,7 @@ jobs:
|
||||
|
||||
package_file="${{ steps.package.outputs.package_file }}"
|
||||
archive_members="$(unzip -Z1 "$package_file" | sort)"
|
||||
expected_members=$'rustfs\nrustfs-cli'
|
||||
expected_members='rustfs'
|
||||
target="${{ matrix.target }}"
|
||||
features="${{ matrix.features }}"
|
||||
if [[ "$target" == *-linux-gnu && "$features" == "pyroscope" ]]; then
|
||||
@@ -691,13 +707,14 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Every Linux target builds natively, so executing the binaries here
|
||||
# musl targets build natively; the gnu legs are same-arch
|
||||
# cargo-zigbuild builds pinned to the release glibc baseline. Either
|
||||
# way the binaries run on their runner, so executing them here
|
||||
# proves each artifact matches its runner architecture.
|
||||
cli_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "$cli_dir"' EXIT
|
||||
unzip -q "$package_file" rustfs rustfs-cli -d "$cli_dir"
|
||||
"$cli_dir/rustfs" --version
|
||||
"$cli_dir/rustfs-cli" --help >/dev/null
|
||||
binary_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "$binary_dir"' EXIT
|
||||
unzip -q "$package_file" rustfs -d "$binary_dir"
|
||||
"$binary_dir/rustfs" --version
|
||||
|
||||
- name: Verify packaged console
|
||||
if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'aarch64-unknown-linux-gnu'
|
||||
|
||||
@@ -57,6 +57,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -94,7 +102,7 @@ jobs:
|
||||
' <<<"$run" >/dev/null
|
||||
jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e '
|
||||
[.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
|
||||
[.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu\\)$"))] as $matches
|
||||
| ($matches | length) == 1 and $matches[0].conclusion == "success"
|
||||
' <<<"$jobs" >/dev/null
|
||||
artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}")
|
||||
@@ -137,7 +145,7 @@ jobs:
|
||||
assert outer.testzip() is None
|
||||
with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
|
||||
entries = inner.infolist()
|
||||
assert {item.filename for item in entries} == {'rustfs', 'rustfs-cli'} and len(entries) == 2
|
||||
assert {item.filename for item in entries} == {'rustfs', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'} and len(entries) == 3
|
||||
assert all(regular(item) and 0 < item.file_size <= 1073741824 for item in entries)
|
||||
assert inner.testzip() is None
|
||||
pathlib.Path('binary').mkdir()
|
||||
@@ -162,7 +170,7 @@ jobs:
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
(cd connect-harness && ./tests/connectivity/diagnostic-schedule-evidence.test.sh)
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_DIAGNOSTIC_SCHEDULE_EVIDENCE="$GITHUB_WORKSPACE/diagnostic-schedule-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=diagnostic-schedule
|
||||
|
||||
@@ -57,6 +57,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -127,7 +135,7 @@ jobs:
|
||||
' <<<"$run" >/dev/null
|
||||
jobs=$(gh api --paginate --slurp "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e '
|
||||
[.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
|
||||
[.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu\\)$"))] as $matches
|
||||
| ($matches | length) == 1 and $matches[0].conclusion == "success"
|
||||
' <<<"$jobs" >/dev/null
|
||||
artifact=$(gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}")
|
||||
@@ -172,7 +180,7 @@ jobs:
|
||||
assert outer.testzip() is None
|
||||
with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
|
||||
entries = inner.infolist()
|
||||
assert {entry.filename for entry in entries} == {'rustfs', 'rustfs-cli', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'} and len(entries) == 4
|
||||
assert {entry.filename for entry in entries} == {'rustfs', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'} and len(entries) == 3
|
||||
assert all(regular(entry) and 0 < entry.file_size <= 1073741824 for entry in entries)
|
||||
assert inner.testzip() is None
|
||||
pathlib.Path('binary').mkdir()
|
||||
@@ -199,7 +207,7 @@ jobs:
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
(cd connect-harness && ./tests/connectivity/health-service-evidence.test.sh)
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_HEALTH_SERVICE_EVIDENCE="$GITHUB_WORKSPACE/health-service-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=health-service
|
||||
|
||||
@@ -73,6 +73,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -107,7 +115,7 @@ jobs:
|
||||
[[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
|
||||
[[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
|
||||
|
||||
expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
|
||||
expected_job='Build RustFS (linux-x86_64-gnu)'
|
||||
jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e --arg name "$expected_job" '
|
||||
[.[].jobs[] | select(.name == $name)] as $matches
|
||||
@@ -154,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_PERFORMANCE_DRIVE_EVIDENCE="$GITHUB_WORKSPACE/performance-drive-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=performance-drive
|
||||
|
||||
@@ -73,6 +73,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -107,7 +115,7 @@ jobs:
|
||||
[[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
|
||||
[[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
|
||||
|
||||
expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
|
||||
expected_job='Build RustFS (linux-x86_64-gnu)'
|
||||
jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e --arg name "$expected_job" '
|
||||
[.[].jobs[] | select(.name == $name)] as $matches
|
||||
@@ -154,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_PERFORMANCE_NETWORK_EVIDENCE="$GITHUB_WORKSPACE/performance-network-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=performance-network
|
||||
|
||||
@@ -77,6 +77,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -118,7 +126,7 @@ jobs:
|
||||
[[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
|
||||
[[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
|
||||
|
||||
expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
|
||||
expected_job='Build RustFS (linux-x86_64-gnu)'
|
||||
jobs=$(github_api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e --arg name "$expected_job" '
|
||||
[.jobs[] | select(.name == $name)] as $matches
|
||||
@@ -173,7 +181,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_CPU_CATALOG="$GITHUB_WORKSPACE/binary/rustfs.cpu-symbol-catalog.json" \
|
||||
CONNECT_E2E_CPU_CATALOG_SHA256="$CATALOG_SHA256" \
|
||||
|
||||
@@ -57,6 +57,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -127,7 +135,7 @@ jobs:
|
||||
' <<<"$run" >/dev/null
|
||||
jobs=$(gh api --paginate --slurp "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e '
|
||||
[.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
|
||||
[.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu\\)$"))] as $matches
|
||||
| ($matches | length) == 1 and $matches[0].conclusion == "success"
|
||||
' <<<"$jobs" >/dev/null
|
||||
artifact=$(gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}")
|
||||
@@ -172,7 +180,7 @@ jobs:
|
||||
assert outer.testzip() is None
|
||||
with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
|
||||
entries = inner.infolist()
|
||||
expected = {'rustfs', 'rustfs-cli', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'}
|
||||
expected = {'rustfs', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'}
|
||||
assert {entry.filename for entry in entries} == expected and len(entries) == len(expected)
|
||||
assert all(regular(entry) and 0 < entry.file_size <= 1073741824 for entry in entries)
|
||||
assert inner.testzip() is None
|
||||
@@ -198,7 +206,7 @@ jobs:
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
(cd connect-harness && ./tests/connectivity/profile-memory-evidence.test.sh)
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_PROFILE_MEMORY_EVIDENCE="$GITHUB_WORKSPACE/profile-memory-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=profile-memory
|
||||
|
||||
@@ -81,6 +81,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -148,7 +156,7 @@ jobs:
|
||||
[[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
|
||||
[[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
|
||||
|
||||
expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
|
||||
expected_job='Build RustFS (linux-x86_64-gnu)'
|
||||
jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e --arg name "$expected_job" '
|
||||
[.[].jobs[] | select(.name == $name)] as $matches
|
||||
@@ -201,7 +209,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_PROFILE_THREADS_EVIDENCE="$GITHUB_WORKSPACE/profile-threads-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=profile-threads
|
||||
|
||||
@@ -73,6 +73,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -107,7 +115,7 @@ jobs:
|
||||
[[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
|
||||
[[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
|
||||
|
||||
expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
|
||||
expected_job='Build RustFS (linux-x86_64-gnu)'
|
||||
jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e --arg name "$expected_job" '
|
||||
[.[].jobs[] | select(.name == $name)] as $matches
|
||||
@@ -154,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_TOP_API_EVIDENCE="$GITHUB_WORKSPACE/top-api-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=top-api
|
||||
|
||||
@@ -68,6 +68,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -138,7 +146,7 @@ jobs:
|
||||
' <<<"$run" >/dev/null
|
||||
jobs=$(gh api --paginate --slurp "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e '
|
||||
[.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches
|
||||
[.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu\\)$"))] as $matches
|
||||
| ($matches | length) == 1 and $matches[0].conclusion == "success"
|
||||
' <<<"$jobs" >/dev/null
|
||||
artifact=$(gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}")
|
||||
@@ -183,7 +191,7 @@ jobs:
|
||||
assert outer.testzip() is None
|
||||
with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner:
|
||||
entries = inner.infolist()
|
||||
expected = {'rustfs', 'rustfs-cli', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'}
|
||||
expected = {'rustfs', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'}
|
||||
assert {entry.filename for entry in entries} == expected and len(entries) == len(expected)
|
||||
assert all(regular(entry) and 0 < entry.file_size <= 1073741824 for entry in entries)
|
||||
assert inner.testzip() is None
|
||||
@@ -213,7 +221,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_TOP_DISK_EVIDENCE="$GITHUB_WORKSPACE/top-disk-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=top-disk
|
||||
|
||||
@@ -73,6 +73,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -107,7 +115,7 @@ jobs:
|
||||
[[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
|
||||
[[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
|
||||
|
||||
expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
|
||||
expected_job='Build RustFS (linux-x86_64-gnu)'
|
||||
jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e --arg name "$expected_job" '
|
||||
[.[].jobs[] | select(.name == $name)] as $matches
|
||||
@@ -154,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_TOP_LOCKS_EVIDENCE="$GITHUB_WORKSPACE/top-locks-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=top-locks
|
||||
|
||||
@@ -73,6 +73,14 @@ jobs:
|
||||
ref: ${{ inputs.connect_sha }}
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Set up Docker Compose
|
||||
uses: ./.github/actions/setup-docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
@@ -107,7 +115,7 @@ jobs:
|
||||
[[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]]
|
||||
[[ $(jq -r '.name' <<<"$run") == "Build and Release" ]]
|
||||
|
||||
expected_job='Build RustFS (linux-x86_64-gnu, sm-standard-4, x86_64-unknown-linux-gnu, false, linux, pyroscope)'
|
||||
expected_job='Build RustFS (linux-x86_64-gnu)'
|
||||
jobs=$(gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100")
|
||||
jq -e --arg name "$expected_job" '
|
||||
[.[].jobs[] | select(.name == $name)] as $matches
|
||||
@@ -154,7 +162,7 @@ jobs:
|
||||
npm --prefix connect-harness/web ci
|
||||
connect-harness/web/node_modules/.bin/playwright install --with-deps chromium
|
||||
make -C connect-harness e2e-connected-dispatch-check
|
||||
RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
TMPDIR="$GITHUB_WORKSPACE" RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \
|
||||
RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \
|
||||
CONNECT_E2E_TOP_RPC_EVIDENCE="$GITHUB_WORKSPACE/top-rpc-service-job-evidence.json" \
|
||||
make -C connect-harness e2e-connected E2E_SCENARIO=top-rpc
|
||||
|
||||
@@ -80,13 +80,24 @@ jobs:
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
cache-shared-key: build-x86_64-unknown-linux-gnu
|
||||
cache-shared-key: build-x86_64-unknown-linux-gnu-glibc-2.31
|
||||
cache-save-if: 'false'
|
||||
install-cross-tools: 'true'
|
||||
target: x86_64-unknown-linux-gnu
|
||||
install-build-packaging-tools: 'false'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Build RustFS
|
||||
run: cargo build --release --locked --target x86_64-unknown-linux-gnu -p rustfs --bins
|
||||
# Pin the glibc baseline (`<target>.<glibc>` for cargo-zigbuild) so
|
||||
# nightlies run on older distros such as RHEL/Rocky 9 (glibc 2.34); #8062.
|
||||
run: cargo zigbuild --release --locked --target x86_64-unknown-linux-gnu.2.31 -p rustfs --bins
|
||||
|
||||
- name: Verify glibc symbol baseline
|
||||
run: |
|
||||
python3 scripts/release/check_glibc.py 2.31 \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs \
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs-cli
|
||||
target/x86_64-unknown-linux-gnu/release/rustfs --version | grep -Fx 'build target : x86_64-unknown-linux-gnu'
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
|
||||
@@ -90,3 +90,4 @@ worktrees/*
|
||||
# insta scratch files; the accepted .snap files ARE the assertions and are committed
|
||||
*.snap.new
|
||||
*.pending-snap
|
||||
.mimosa
|
||||
Generated
+292
-256
File diff suppressed because it is too large
Load Diff
+76
-75
@@ -74,7 +74,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.99.0"
|
||||
version = "1.0.1"
|
||||
version = "1.0.2"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -91,56 +91,56 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.1" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.1" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.1" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.1" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.1" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.1" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.1" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.1" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.1" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.1" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.1" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.1" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.1" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.1" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.1" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.1" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.1" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.1" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.1", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.1" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.1" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.1" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.1" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.1" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.1" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.1" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.1" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.1" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.1" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.1" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.1" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.1" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.1" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.1" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.1" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.1" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.2" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.2" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.2" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.2" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.2" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.2" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.2" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.2" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.2" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.2" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.2" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.2" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.2" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.2" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.2" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.2" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.2" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.2" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.2" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.2" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.2" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.2" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.2" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.2" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.2" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.2", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.2" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.2" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.2" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.2" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.2" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.2" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.2" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.2" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.2" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.2" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.2" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.2" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.2" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.2" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.2" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.2" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.2" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.2" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.2" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.2" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.2" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.2" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.2" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.2" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -156,8 +156,9 @@ futures-lite = "2.6.1"
|
||||
futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.9.0" }
|
||||
lapin = { default-features = false, version = "4.12.0" }
|
||||
lapin = { default-features = false, version = "4.12.2" }
|
||||
hyper = { version = "1.12.0" }
|
||||
h2 = { version = "0.4.20" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.10" }
|
||||
hyper-util = { version = "0.1.21" }
|
||||
http = "1.5.0"
|
||||
@@ -168,11 +169,11 @@ minlz = "1.2.3"
|
||||
reqwest = "0.13.5"
|
||||
rustfs-kafka-async = { version = "1.4.0" }
|
||||
socket2 = { version = "0.6.5" }
|
||||
tokio = { version = "1.53.2" }
|
||||
tokio = { version = "1.53.3" }
|
||||
tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
tokio-util = { version = "0.7.19" }
|
||||
tokio-util = { version = "0.7.20" }
|
||||
tonic = { version = "0.14.6" }
|
||||
tonic-prost = { version = "0.14.6" }
|
||||
tonic-prost-build = { version = "0.14.6" }
|
||||
@@ -180,7 +181,7 @@ tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
h3-quinn = "0.0.10"
|
||||
quinn = { version = "0.11.11", default-features = false, features = [
|
||||
quinn = { version = "0.11.12", default-features = false, features = [
|
||||
"futures-io",
|
||||
"runtime-tokio",
|
||||
"rustls-aws-lc-rs",
|
||||
@@ -200,7 +201,7 @@ rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde_ignored = { version = "0.1" }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
|
||||
serde_json = { version = "1.0.151" }
|
||||
serde_json = { version = "1.0.154" }
|
||||
serde_urlencoded = "0.7.1"
|
||||
|
||||
# Cryptography and Security
|
||||
@@ -224,7 +225,7 @@ rustls-native-certs = "0.8"
|
||||
rustls-pki-types = "1.15.1"
|
||||
x509-parser = "0.18.1"
|
||||
sha1 = "0.11.0"
|
||||
sha2 = "0.11.0"
|
||||
sha2 = "0.11.1"
|
||||
subtle = "2.6"
|
||||
zeroize = { version = "1.9.1" }
|
||||
proptest = "1"
|
||||
@@ -268,7 +269,7 @@ crossbeam-queue = "0.3.14"
|
||||
crossbeam-channel = "0.5.17"
|
||||
crossbeam-deque = "0.8.8"
|
||||
crossbeam-utils = "0.8.23"
|
||||
datafusion = { default-features = false, version = "55.1.0" }
|
||||
datafusion = { default-features = false, version = "55.2.0" }
|
||||
derive_builder = "0.20.2"
|
||||
ed25519-dalek = "3.0.0"
|
||||
enumset = "1.1.14"
|
||||
@@ -319,14 +320,14 @@ rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.7.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.5" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
|
||||
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.4"
|
||||
smallvec = { version = "1.16.2" }
|
||||
smallvec = { version = "1.16.3" }
|
||||
compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
@@ -340,12 +341,12 @@ tracing = { version = "0.1.44" }
|
||||
tracing-appender = "0.2.5"
|
||||
tracing-core = "0.1.36"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = { version = "0.34" }
|
||||
tracing-opentelemetry = { version = "0.34.0" }
|
||||
tracing-subscriber = { version = "0.3.23" }
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.8"
|
||||
urlencoding = "2.1.3"
|
||||
uuid = { version = "1.27.0" }
|
||||
uuid = { version = "1.28.0" }
|
||||
vaultrs = { version = "0.8.0" }
|
||||
tar = "0.4.46"
|
||||
walkdir = "2.5.0"
|
||||
@@ -353,26 +354,26 @@ winapi-util = "0.1.11"
|
||||
windows = { version = "0.62.2" }
|
||||
windows-sys = "0.61.2"
|
||||
xxhash-rust = { version = "0.8.19" }
|
||||
zip = "8.6.0"
|
||||
zstd = "0.14.0"
|
||||
zip = "9.0.3"
|
||||
zstd = "0.14.2"
|
||||
|
||||
# Observability and Metrics
|
||||
metrics = "0.24.6"
|
||||
metrics-util = "0.20"
|
||||
dial9 = { version = "0.5.3", default-features = false }
|
||||
opentelemetry = { version = "0.33.0" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.0" }
|
||||
opentelemetry-otlp = { version = "0.33.0" }
|
||||
opentelemetry-proto = { version = "0.33.0", default-features = false, features = ["metrics", "gen-tonic-messages"] }
|
||||
opentelemetry_sdk = { version = "0.33.0" }
|
||||
opentelemetry-semantic-conventions = { version = "0.33.0" }
|
||||
opentelemetry-stdout = { version = "0.33.0" }
|
||||
pyroscope = { version = "2.1.1" }
|
||||
dial9 = { version = "0.5.4", default-features = false }
|
||||
opentelemetry = { version = "0.33.1" }
|
||||
opentelemetry-appender-tracing = { version = "0.33.1" }
|
||||
opentelemetry-otlp = { version = "0.33.1" }
|
||||
opentelemetry-proto = { version = "0.33.1", default-features = false, features = ["metrics", "gen-tonic-messages"] }
|
||||
opentelemetry_sdk = { version = "0.33.1" }
|
||||
opentelemetry-semantic-conventions = { version = "0.33.1" }
|
||||
opentelemetry-stdout = { version = "0.33.1" }
|
||||
pyroscope = { version = "2.1.2" }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.1" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "12.1.1" }
|
||||
suppaftp = { version = "12.2.0" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.64.1" }
|
||||
russh-sftp = "3.0.1"
|
||||
@@ -383,7 +384,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.5.6" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.28.5", default-features = false }
|
||||
hotpath = { version = "0.28.6", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
@@ -51,6 +51,8 @@ unparseable = "unparseable"
|
||||
# Disaster-recovery objectives: recovery time and recovery point.
|
||||
RTO = "RTO"
|
||||
rto = "rto"
|
||||
# rustix OFlags variant (POSIX O_WRONLY equivalent).
|
||||
WRONLY = "WRONLY"
|
||||
|
||||
[files]
|
||||
extend-exclude = []
|
||||
|
||||
@@ -51,6 +51,11 @@ pub const ENV_OBS_PROFILING_EXPORT_ENABLED: &str = "RUSTFS_OBS_PROFILING_EXPORT_
|
||||
/// PUT records multiple timers and histograms when attribution is active.
|
||||
pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED";
|
||||
|
||||
/// Enables sampled in-process GetObject stage summaries without OTLP histograms.
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED";
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE";
|
||||
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS";
|
||||
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
|
||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||
@@ -159,6 +164,15 @@ mod tests {
|
||||
assert_eq!(ENV_OBS_LOGS_EXPORT_ENABLED, "RUSTFS_OBS_LOGS_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PROFILING_EXPORT_ENABLED, "RUSTFS_OBS_PROFILING_EXPORT_ENABLED");
|
||||
assert_eq!(ENV_OBS_PUT_STAGE_METRICS_ENABLED, "RUSTFS_OBS_PUT_STAGE_METRICS_ENABLED");
|
||||
assert_eq!(ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED, "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED");
|
||||
assert_eq!(
|
||||
ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE,
|
||||
"RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE"
|
||||
);
|
||||
assert_eq!(
|
||||
ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS,
|
||||
"RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS"
|
||||
);
|
||||
// Test log cleanup related env keys
|
||||
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
|
||||
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
|
||||
|
||||
@@ -860,10 +860,12 @@ pub(crate) async fn start_rebalance(cluster: &RustFSTestClusterEnvironment) -> T
|
||||
if status.is_success() {
|
||||
break response;
|
||||
}
|
||||
if status == StatusCode::INTERNAL_SERVER_ERROR
|
||||
&& response.contains("requires a live fleet capability proof")
|
||||
&& Instant::now() < deadline
|
||||
{
|
||||
// The init stage answers 503 + Retry-After while the cluster capability
|
||||
// proof is still missing; older binaries surfaced the same condition as
|
||||
// a 500 carrying the proof marker.
|
||||
let waiting_for_readiness = status == StatusCode::SERVICE_UNAVAILABLE
|
||||
|| (status == StatusCode::INTERNAL_SERVER_ERROR && response.contains("requires a live fleet capability proof"));
|
||||
if waiting_for_readiness && Instant::now() < deadline {
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1969,11 +1969,16 @@ mod tests {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("pre-interrupt background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if background_enabled || only_admin_heal_is_active(&status) {
|
||||
let admin_active = background_rejoin_heal_evidence
|
||||
|| (status["state"].as_str() == Some("active")
|
||||
&& status["healOperations"]["activeBySource"]["admin"].as_u64() == Some(1));
|
||||
if admin_active && (background_enabled || only_admin_heal_is_active(&status)) {
|
||||
break status;
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!("non-admin heal did not drain before the interruption deadline: {status}").into());
|
||||
return Err(
|
||||
format!("heal tasks did not reach the required interruption state before the deadline: {status}").into(),
|
||||
);
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
};
|
||||
@@ -2633,4 +2638,157 @@ mod tests {
|
||||
};
|
||||
assert!(!outage_candidate_targets_replacement(&insufficient_peer_shards, 12, replacement_set_slot));
|
||||
}
|
||||
async fn two_node_heal_status(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
bucket: Option<&str>,
|
||||
recreate: bool,
|
||||
scan_mode: u8,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/heal/{}", cluster.nodes[0].url, bucket.unwrap_or_default());
|
||||
let options = serde_json::json!({
|
||||
"recursive": true, "dryRun": false, "remove": false,
|
||||
"recreate": recreate, "scanMode": scan_mode, "updateParity": false, "nolock": false,
|
||||
});
|
||||
let started =
|
||||
start_root_heal_when_control_ready(&url, &options.to_string(), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let started: serde_json::Value = serde_json::from_str(&started)?;
|
||||
let token = started["clientToken"]
|
||||
.as_str()
|
||||
.ok_or_else(|| format!("missing heal token: {started}"))?;
|
||||
let status_url = format!("{url}?clientToken={token}");
|
||||
let deadline = Instant::now() + Duration::from_secs(240);
|
||||
loop {
|
||||
let body = timeout(
|
||||
Duration::from_secs(20),
|
||||
signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await??;
|
||||
let status: serde_json::Value = serde_json::from_str(&body)?;
|
||||
match status["summary"].as_str() {
|
||||
Some("finished" | "stopped") => return Ok(status),
|
||||
Some("running") if Instant::now() < deadline => sleep(Duration::from_millis(250)).await,
|
||||
_ => return Err(format!("heal did not reach a terminal state: {status}").into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn verify_two_node_directory_recovery(
|
||||
remove_bucket: bool,
|
||||
scan_mode: u8,
|
||||
bucket_scoped: bool,
|
||||
recreate: bool,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(2).await?;
|
||||
// Isolate explicit administrator recovery from scanner, MRF, and disk healing.
|
||||
for key in [
|
||||
"RUSTFS_HEAL_AUTO_HEAL_ENABLE",
|
||||
"RUSTFS_HEAL_MRF_ENABLE",
|
||||
"RUSTFS_SCANNER_ENABLED",
|
||||
"RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE",
|
||||
] {
|
||||
cluster.set_env(key, "false");
|
||||
}
|
||||
cluster.start().await?;
|
||||
let source = cluster.create_s3_client(0)?;
|
||||
let target = cluster.create_s3_client(1)?;
|
||||
let bucket = "two-node-directory-recovery";
|
||||
source.create_bucket().bucket(bucket).send().await?;
|
||||
let objects = [
|
||||
("empty", Vec::new()),
|
||||
("nested/inline", vec![0x3c; 1024]),
|
||||
("nested/external", (0_u8..=255).cycle().take(1024 * 1024).collect::<Vec<_>>()),
|
||||
];
|
||||
for (key, bytes) in &objects {
|
||||
source
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(*key)
|
||||
.body(ByteStream::from(bytes.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
let target_bucket = Path::new(&cluster.nodes[1].data_dir).join(bucket);
|
||||
let original = objects
|
||||
.iter()
|
||||
.map(|(key, _)| census_object_version_on_disk(Path::new(&cluster.nodes[1].data_dir), bucket, key, None))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
for census in &original {
|
||||
assert!(census.has_xl_meta, "fixture must have a target metadata copy: {census:?}");
|
||||
}
|
||||
assert!(
|
||||
!original[2].present_part_fingerprints.is_empty(),
|
||||
"large fixture must exercise external parts"
|
||||
);
|
||||
let removed = Path::new(&cluster.temp_dir).join("removed");
|
||||
tokio::fs::create_dir(&removed).await?;
|
||||
if remove_bucket {
|
||||
tokio::fs::rename(&target_bucket, removed.join(bucket)).await?;
|
||||
} else {
|
||||
for (index, (key, _)) in objects.iter().enumerate() {
|
||||
tokio::fs::rename(target_bucket.join(key), removed.join(index.to_string())).await?;
|
||||
}
|
||||
}
|
||||
let status = two_node_heal_status(&cluster, bucket_scoped.then_some(bucket), recreate, scan_mode).await?;
|
||||
assert_eq!(status["summary"], "finished", "administrator recovery should complete: {status}");
|
||||
assert_eq!(status["outcome"]["execution"]["state"], "completed");
|
||||
assert_eq!(status["outcome"]["coverage"], "complete");
|
||||
for ((key, _), before) in objects.iter().zip(&original) {
|
||||
let after = census_object_version_on_disk(Path::new(&cluster.nodes[1].data_dir), bucket, key, None)?;
|
||||
assert!(after.has_xl_meta, "metadata must be restored before GET: {key}");
|
||||
assert_eq!(
|
||||
after.present_part_fingerprints, before.present_part_fingerprints,
|
||||
"external shard bytes must match: {key}"
|
||||
);
|
||||
assert_eq!(
|
||||
after.inline_data_fingerprint, before.inline_data_fingerprint,
|
||||
"inline shard bytes must match: {key}"
|
||||
);
|
||||
}
|
||||
cluster.stop_node(0)?;
|
||||
for (key, expected) in &objects {
|
||||
let actual = timeout(Duration::from_secs(30), async {
|
||||
target
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(*key)
|
||||
.send()
|
||||
.await?
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.map(|body| body.into_bytes())
|
||||
.map_err(Box::<dyn Error + Send + Sync>::from)
|
||||
})
|
||||
.await??;
|
||||
assert_eq!(actual.len(), expected.len(), "target-only read length: {key}");
|
||||
assert_eq!(sha256_hex(&actual), sha256_hex(expected), "target-only content: {key}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_bucket_discovery_and_explicit_recreate() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(true, 2, true, true).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_missing_bucket_default_cluster_recovery() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(true, 2, false, false).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_missing_bucket_default_scoped_recovery() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(true, 2, true, false).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_object_directory_recovery_normal() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(false, 1, false, false).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_two_node_object_directory_recovery_deep() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
verify_two_node_directory_recovery(false, 2, false, false).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,6 +195,91 @@ async fn wait_for_quorum_read_admission(clients: &[Client], bucket: &str) -> Tes
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cold_degraded_object_mutations_require_only_storage_quorum() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_OBS_METRICS_EXPORT_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_GET_OBJECT_METADATA_CACHE_DISTRIBUTED_ENABLE", "false");
|
||||
cluster.start().await?;
|
||||
let clients = cluster
|
||||
.create_all_clients()?
|
||||
.into_iter()
|
||||
.map(|client| {
|
||||
Client::from_conf(
|
||||
client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1))
|
||||
.build(),
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let bucket = "cold-degraded-mutations";
|
||||
clients[0].create_bucket().bucket(bucket).send().await?;
|
||||
// No object mutation may warm a process-wide configuration probe before
|
||||
// the missing peer is introduced. Three shards still satisfy EC:2 writes.
|
||||
cluster.stop_node(3)?;
|
||||
let admission_deadline = Instant::now() + Duration::from_secs(30);
|
||||
for client in &clients[..3] {
|
||||
loop {
|
||||
match client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("never-created-readiness-probe")
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Err(error) if error.as_service_error().and_then(|error| error.meta().code()) == Some("NoSuchKey") => break,
|
||||
Err(error)
|
||||
if error.raw_response().is_some_and(|response| response.status().as_u16() == 503)
|
||||
&& Instant::now() < admission_deadline =>
|
||||
{
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
Ok(_) => panic!("a cold readiness probe must not create an object"),
|
||||
}
|
||||
}
|
||||
}
|
||||
let body = b"first write while one peer is offline";
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("object")
|
||||
.body(Bytes::from_static(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
for client in &clients[..3] {
|
||||
assert_quorum_object_body(client, bucket, "object", body).await?;
|
||||
}
|
||||
// This coordinator has not previously committed an object mutation either.
|
||||
clients[1].delete_object().bucket(bucket).key("object").send().await?;
|
||||
let error = clients[2]
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("acknowledged deletion must be visible at another entrypoint");
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("NoSuchKey"));
|
||||
|
||||
cluster.stop_node(2)?;
|
||||
let error = clients[0]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("below-quorum")
|
||||
.body(Bytes::from_static(body).into())
|
||||
.send()
|
||||
.await
|
||||
.expect_err("removing the cache gate must not weaken storage write quorum");
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
cluster.stop();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_degraded_cluster_read_quorum_follows_erasure_layout() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
@@ -3945,7 +3945,15 @@ async fn test_bucket_replication_version_purge_of_non_inline_object_releases_sou
|
||||
(sse_key, sse_put.version_id().ok_or("SSE PUT omitted version ID")?.to_string()),
|
||||
(large_key, large_put.version_id().ok_or("large PUT omitted version ID")?.to_string()),
|
||||
];
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
// Managed-SSE listings may omit ETags. Check the replicated version's body
|
||||
// before deleting it rather than comparing keyless listing fingerprints.
|
||||
for (key, version_id) in &purged {
|
||||
let source_body = get_version_body(&source_client, &source_bucket, key, version_id).await?;
|
||||
let expected_sha256: [u8; 32] = Sha256::digest(&source_body).into();
|
||||
wait_for_replicated_sha256(&target_client, &target_bucket, key, expected_sha256).await?;
|
||||
let target_body = get_version_body(&target_client, &target_bucket, key, version_id).await?;
|
||||
assert_eq!(target_body, source_body, "replicated version body differs for {key}/{version_id}");
|
||||
}
|
||||
|
||||
for (key, version_id) in &purged {
|
||||
source_client
|
||||
@@ -3956,10 +3964,6 @@ async fn test_bucket_replication_version_purge_of_non_inline_object_releases_sou
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
assert_replication_converged(&source_client, &source_bucket, &target_client, &target_bucket).await?;
|
||||
let target_state = list_replication_state(&target_client, &target_bucket).await?;
|
||||
assert!(target_state.is_empty(), "target retained an explicitly purged version: {target_state:?}");
|
||||
|
||||
// The purge state is applied on the source asynchronously after the target
|
||||
// acknowledges the delete; only then does the retained version go away and
|
||||
// the bucket become deletable. A listing that is empty while DeleteBucket
|
||||
@@ -3984,6 +3988,9 @@ async fn test_bucket_replication_version_purge_of_non_inline_object_releases_sou
|
||||
}
|
||||
}
|
||||
|
||||
let target_state = list_replication_state(&target_client, &target_bucket).await?;
|
||||
assert!(target_state.is_empty(), "target retained an explicitly purged version: {target_state:?}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -183,6 +183,7 @@ sha2 = { workspace = true }
|
||||
hex-simd = { workspace = true }
|
||||
tempfile.workspace = true
|
||||
hyper = { workspace = true, features = ["http2", "http1", "server"] }
|
||||
h2 = { workspace = true }
|
||||
hyper-util = { workspace = true, features = ["tokio", "server-auto", "server-graceful", "tracing"] }
|
||||
hyper-rustls = { workspace = true, default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "aws-lc-rs"] }
|
||||
hostname.workspace = true
|
||||
@@ -200,8 +201,8 @@ md-5.workspace = true
|
||||
memmap2 = { workspace = true }
|
||||
libc.workspace = true
|
||||
# "process" adds getrlimit for the io_uring fd-cache RLIMIT_NOFILE gate
|
||||
# (backlog#1178); "fs" comes from the workspace default.
|
||||
rustix = { workspace = true, features = ["process", "fs"] }
|
||||
# (backlog#1178); "thread" provides cleanup affinity; "fs" is a workspace default.
|
||||
rustix = { workspace = true, features = ["process", "fs", "thread"] }
|
||||
rustfs-madmin.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
|
||||
@@ -425,10 +425,11 @@ pub mod disk {
|
||||
BATCH_READ_VERSION_MAX_ITEMS, BUCKET_META_PREFIX, BatchReadVersionItem, BatchReadVersionReq, BatchReadVersionResp,
|
||||
CheckPartsResp, ConditionalFileUpdate, DELETE_RECOVERY_PROTOCOL_VERSION, DeleteOptions, DeleteRecoveryAction,
|
||||
DeleteRecoveryCursor, DeleteRecoveryDigest, DeleteRecoveryInspection, DeleteRecoveryPage, DeleteRecoveryRpcOperation,
|
||||
DeleteRecoveryRpcReply, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption, DiskStore, FileInfoVersions,
|
||||
FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize, PartTransactionAction,
|
||||
RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE,
|
||||
SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk, validate_batch_read_version_item_count,
|
||||
DeleteRecoveryRpcReply, DeleteRecoveryState, Disk, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskOption,
|
||||
DiskStore, FileInfoVersions, FileReader, FileWriter, HEALING_MARKER_PATH, NsScannerOpenRequest, OldCurrentSize,
|
||||
PartTransactionAction, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
||||
STORAGE_FORMAT_FILE, SnapshotLeaseToken, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
|
||||
@@ -2417,6 +2417,16 @@ fn is_slow_down(err: &Error) -> bool {
|
||||
matches!(err, Error::SlowDown)
|
||||
}
|
||||
|
||||
/// Lifecycle expiry is idempotent when the exact object or version has already
|
||||
/// disappeared between the scanner listing and the delete admission.
|
||||
///
|
||||
/// Keep this predicate narrower than bucket/volume "not found" helpers: an
|
||||
/// unavailable bucket or disk is an operational failure and must not be
|
||||
/// reported as a completed expiry.
|
||||
fn is_lifecycle_expiry_already_gone(err: &Error) -> bool {
|
||||
is_err_object_not_found(err) || is_err_version_not_found(err)
|
||||
}
|
||||
|
||||
/// Resolves the expiry worker count from the single documented knob,
|
||||
/// `RUSTFS_MAX_EXPIRY_WORKERS`: a set, parsable, non-zero value wins;
|
||||
/// anything else falls back to `min(cpus, 16)`. The historical
|
||||
@@ -4920,6 +4930,20 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
//audit_log_lifecycle(*oi, ILMExpiry, tags, traceFn);
|
||||
Ok(dobj)
|
||||
}
|
||||
Err(err) if is_lifecycle_expiry_already_gone(&err) => {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_DELETE_COMPLETED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
operation = "delete_restored_object",
|
||||
reason = "already_gone",
|
||||
"Lifecycle delete target was already absent"
|
||||
);
|
||||
crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
|
||||
Ok(oi.clone())
|
||||
}
|
||||
Err(err) => Err(std::io::Error::other(err)),
|
||||
};
|
||||
}
|
||||
@@ -4930,6 +4954,8 @@ async fn expire_transitioned_object_with_lock_lost_signal(
|
||||
// pointing to a remote tier version that has already been removed. If this
|
||||
// only creates a delete marker, remote cleanup must be driven by persisted
|
||||
// free-version recovery rather than the visible delete result.
|
||||
// Keep remote transitioned not-found errors visible: local metadata
|
||||
// absence alone does not prove that the tier object was cleaned up.
|
||||
let dobj = match api.delete_object(&oi.bucket, &oi.name, opts).await {
|
||||
Ok(obj) => obj,
|
||||
Err(e) => {
|
||||
@@ -5802,6 +5828,20 @@ async fn apply_expiry_on_non_transitioned_objects_with_lock_lost_signal(
|
||||
.await
|
||||
{
|
||||
Ok(dobj) => dobj,
|
||||
Err(err) if is_lifecycle_expiry_already_gone(&err) => {
|
||||
debug!(
|
||||
event = EVENT_LIFECYCLE_DELETE_COMPLETED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
bucket = %oi.bucket,
|
||||
object = %oi.name,
|
||||
operation = "delete_object",
|
||||
reason = "already_gone",
|
||||
"Lifecycle delete target was already absent"
|
||||
);
|
||||
crate::object_api::notify_object_mutation(&oi.bucket, &oi.name).await;
|
||||
return true;
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
event = EVENT_LIFECYCLE_DELETE_FAILED,
|
||||
@@ -6041,18 +6081,19 @@ mod tests {
|
||||
cleanup_empty_multipart_sha_dirs_on_local_disks, cleanup_stale_multipart_uploads_once_at,
|
||||
enqueue_recovered_free_version_with_state, enqueue_transition_for_existing_objects_scoped,
|
||||
enqueue_transition_with_lifecycle, enqueue_transition_with_lifecycle_report, eval_action_from_lifecycle,
|
||||
get_lock_acquire_timeout, jitter_tier_free_version_recovery_delay, lifecycle_action_blocked_by_replication,
|
||||
lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action,
|
||||
lifecycle_rule_has_date_expiration, lifecycle_version_delete_target, manual_transition_duration_elapsed,
|
||||
manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink, manual_transition_version_marker,
|
||||
manual_transition_worker_failure_reason, mark_delete_opts_skip_decommissioned_on_remote_success,
|
||||
merge_stale_multipart_candidate, persist_manual_transition_job_progress_if_owned,
|
||||
persist_manual_transition_page_checkpoint, recover_manual_transition_job, recover_manual_transition_jobs,
|
||||
resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout,
|
||||
resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator,
|
||||
run_tier_free_version_recovery_loop, select_restore_s3_location, set_lifecycle_observability_observer,
|
||||
set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update,
|
||||
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
|
||||
get_lock_acquire_timeout, is_lifecycle_expiry_already_gone, jitter_tier_free_version_recovery_delay,
|
||||
lifecycle_action_blocked_by_replication, lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object,
|
||||
lifecycle_replication_blocks_action, lifecycle_rule_has_date_expiration, lifecycle_version_delete_target,
|
||||
manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink,
|
||||
manual_transition_version_marker, manual_transition_worker_failure_reason,
|
||||
mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate,
|
||||
persist_manual_transition_job_progress_if_owned, persist_manual_transition_page_checkpoint,
|
||||
recover_manual_transition_job, recover_manual_transition_jobs, resolve_tier_free_version_recovery_enabled,
|
||||
resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count,
|
||||
resolve_transition_workers_absolute_max, run_tier_free_version_recovery_coordinator, run_tier_free_version_recovery_loop,
|
||||
select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
||||
should_defer_date_expiry_for_recent_config_update, transitioned_cleanup_tuple, transitioned_object_delete_opts,
|
||||
wait_for_tier_free_version_recovery,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use super::{delete_free_version_remote_object_then, encode_dir_object, get_transitioned_object_reader_with_tier_manager};
|
||||
@@ -10511,6 +10552,26 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_expiry_already_gone_only_accepts_object_or_version_absence() {
|
||||
assert!(is_lifecycle_expiry_already_gone(&Error::FileNotFound));
|
||||
assert!(is_lifecycle_expiry_already_gone(&Error::ObjectNotFound(
|
||||
"bucket".to_string(),
|
||||
"object".to_string(),
|
||||
)));
|
||||
assert!(is_lifecycle_expiry_already_gone(&Error::FileVersionNotFound));
|
||||
assert!(is_lifecycle_expiry_already_gone(&Error::VersionNotFound(
|
||||
"bucket".to_string(),
|
||||
"object".to_string(),
|
||||
"version".to_string(),
|
||||
)));
|
||||
|
||||
assert!(!is_lifecycle_expiry_already_gone(&Error::VolumeNotFound));
|
||||
assert!(!is_lifecycle_expiry_already_gone(&Error::DiskNotFound));
|
||||
assert!(!is_lifecycle_expiry_already_gone(&Error::ErasureReadQuorum));
|
||||
assert!(!is_lifecycle_expiry_already_gone(&Error::MethodNotAllowed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_transition_counts_already_transitioned_object() {
|
||||
let lc = latest_transition_lifecycle();
|
||||
@@ -13385,7 +13446,9 @@ mod tests {
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
for (attempt_incarnation, expected) in [(Uuid::new_v4(), false), (incarnation, true), (incarnation, false)] {
|
||||
// Replaying the same scanner snapshot after the delete is an
|
||||
// already-gone success and must not be reported as a failure.
|
||||
for (attempt_incarnation, expected) in [(Uuid::new_v4(), false), (incarnation, true), (incarnation, true)] {
|
||||
assert_eq!(
|
||||
super::apply_expiry_on_non_transitioned_objects(
|
||||
ecstore.clone(),
|
||||
|
||||
@@ -178,6 +178,13 @@ pub(crate) fn object_lock_config_state_from_authoritative_metadata(bm: &BucketMe
|
||||
Ok(ObjectLockConfigState::ConfirmedAbsent)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub(crate) struct CachedQuotaMetadataSnapshot {
|
||||
pub(crate) quota: Option<BucketQuota>,
|
||||
pub(crate) bucket_incarnation: Uuid,
|
||||
pub(crate) quota_revision: OffsetDateTime,
|
||||
}
|
||||
|
||||
/// Convert the persisted serving-layer configuration into the storage-level
|
||||
/// [`DefaultRetention`](crate::bucket::object_lock::types::DefaultRetention)
|
||||
/// the WORM evaluation code consumes (rustfs/backlog#1842). A rule without a
|
||||
@@ -518,6 +525,36 @@ pub(crate) async fn get_bucket_incarnation_id_in(ctx: &crate::runtime::instance:
|
||||
sys.get_bucket_incarnation_id_from_disk(bucket).await
|
||||
}
|
||||
|
||||
/// Directory loss is recoverable only for a committed, non-retired generation.
|
||||
/// The caller retains the bucket lifecycle fence through the ensuing reconstruction.
|
||||
pub(crate) async fn validate_bucket_reconstruction_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
expected: Uuid,
|
||||
) -> Result<()> {
|
||||
let api = bucket_metadata_sys_of(ctx)?.read().await.object_store();
|
||||
let lock = api
|
||||
.new_ns_lock(RUSTFS_META_BUCKET, &bucket_metadata_transaction_lock_key(bucket))
|
||||
.await?;
|
||||
let guard = lock.get_read_lock(crate::set_disk::get_lock_acquire_timeout()).await?;
|
||||
let (metadata, persisted) = load_bucket_metadata_parse_with_presence(api.clone(), bucket, true).await?;
|
||||
if !persisted
|
||||
|| !metadata.bucket_incarnation_sidecar
|
||||
|| expected.is_nil()
|
||||
|| metadata.bucket_incarnation_id != expected
|
||||
|| !metadata.bucket_creation_committed
|
||||
{
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
if crate::bucket::retirement::is_retired(api, bucket, expected).await? {
|
||||
return Err(Error::BucketNotFound(bucket.to_owned()));
|
||||
}
|
||||
if guard.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate against disk while retaining an already-held Object Lock fence.
|
||||
pub(crate) async fn get_bucket_incarnation_id_for_options_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
@@ -1450,6 +1487,9 @@ fn quota_config_and_incarnation_from_authority(
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
|
||||
return Err(Error::other("persisted bucket quota configuration is invalid"));
|
||||
}
|
||||
Ok((
|
||||
metadata.quota_config.clone(),
|
||||
metadata.bucket_incarnation_id,
|
||||
@@ -1464,6 +1504,15 @@ fn quota_config_and_incarnation_from_authority(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn get_cached_quota_config_and_incarnation_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
) -> Result<Option<CachedQuotaMetadataSnapshot>> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
bucket_meta_sys.cached_quota_config_and_incarnation(bucket).await
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
|
||||
@@ -2979,6 +3028,27 @@ impl BucketMetadataSys {
|
||||
}
|
||||
}
|
||||
|
||||
async fn cached_quota_config_and_incarnation(&self, bucket: &str) -> Result<Option<CachedQuotaMetadataSnapshot>> {
|
||||
let metadata = {
|
||||
let metadata_map = self.metadata_map.read().await;
|
||||
metadata_map.get(bucket).cloned()
|
||||
};
|
||||
let Some(metadata) = metadata else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !metadata.bucket_incarnation_sidecar || metadata.bucket_incarnation_id.is_nil() {
|
||||
return Ok(None);
|
||||
}
|
||||
if !metadata.quota_config_json.is_empty() && metadata.quota_config.is_none() {
|
||||
return Err(Error::other("persisted bucket quota configuration is invalid"));
|
||||
}
|
||||
Ok(Some(CachedQuotaMetadataSnapshot {
|
||||
quota: metadata.quota_config.clone(),
|
||||
bucket_incarnation: metadata.bucket_incarnation_id,
|
||||
quota_revision: metadata.quota_config_updated_at,
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
|
||||
let (bm, _) = self.get_config(bucket).await?;
|
||||
|
||||
@@ -4883,6 +4953,43 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let sys = bucket_metadata_sys_of(&store.ctx).expect("scoped metadata system");
|
||||
let bucket = "stale-no-quota-admission";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let (stale, _) = sys.read().await.get_config(bucket).await.expect("cache initial metadata");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("publish quota to authoritative storage");
|
||||
// Model a peer that has not received the newer configuration snapshot.
|
||||
sys.read().await.set(bucket.to_owned(), stale).await;
|
||||
assert!(
|
||||
get_cached_quota_config_and_incarnation_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read stale cache")
|
||||
.expect("cached bucket identity")
|
||||
.quota
|
||||
.is_none()
|
||||
);
|
||||
|
||||
temp_env::async_with_vars([("RUSTFS_QUOTA_BEGIN_SAFE_NO_QUOTA_CACHE_FAST_PATH", Some("true"))], async {
|
||||
let error = crate::bucket::quota::reservation::begin(&store.ctx, bucket, "object", &ObjectOptions::default(), 0, 0)
|
||||
.await
|
||||
.err()
|
||||
.expect("persisted quota must reject admission without its snapshot");
|
||||
assert!(matches!(error, Error::PartMissingOrCorrupt), "{error:?}");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn peer_reload_clears_negative_cache_after_remote_bucket_creation() {
|
||||
|
||||
@@ -850,6 +850,11 @@ pub(crate) async fn begin(
|
||||
let snapshot_admission = opts.quota_admission;
|
||||
let data_movement = opts.data_movement;
|
||||
if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
|
||||
meta_bucket_fast_path_started,
|
||||
);
|
||||
return Ok(QuotaContext {
|
||||
store: None,
|
||||
bucket: bucket.to_string(),
|
||||
@@ -906,9 +911,63 @@ pub(crate) async fn begin(
|
||||
});
|
||||
}
|
||||
|
||||
let metadata_lock_started = rustfs_io_metrics::put_stage_timer();
|
||||
let metadata_guard = metadata_sys::acquire_bucket_metadata_transaction_read_lock_for_options_in(ctx, bucket, opts).await?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_METADATA_LOCK,
|
||||
metadata_lock_started,
|
||||
);
|
||||
let cache_lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let cached_quota_snapshot = if rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
match metadata_sys::get_cached_quota_config_and_incarnation_in(ctx, bucket).await {
|
||||
Ok(Some(snapshot)) => {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision(
|
||||
"candidate",
|
||||
quota_cache_reason(snapshot.quota.as_ref()),
|
||||
);
|
||||
Some(snapshot)
|
||||
}
|
||||
Ok(None) => {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "no_authoritative_cache");
|
||||
None
|
||||
}
|
||||
Err(_) => {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("miss", "invalid_cached_quota");
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CACHE_LOOKUP,
|
||||
cache_lookup_started,
|
||||
);
|
||||
let config_read_started = rustfs_io_metrics::put_stage_timer();
|
||||
let (quota, bucket_incarnation, quota_revision) =
|
||||
metadata_sys::get_quota_config_and_incarnation_from_disk_for_options_in(ctx, bucket, opts, &metadata_guard).await?;
|
||||
if let Some(cached) = cached_quota_snapshot.as_ref() {
|
||||
let reason = quota_cache_reason(quota.as_ref());
|
||||
if cached.quota == quota && cached.bucket_incarnation == bucket_incarnation && cached.quota_revision == quota_revision {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_match", reason);
|
||||
} else {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch", reason);
|
||||
if cached.quota != quota {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota");
|
||||
}
|
||||
if cached.bucket_incarnation != bucket_incarnation {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "bucket_incarnation");
|
||||
}
|
||||
if cached.quota_revision != quota_revision {
|
||||
rustfs_io_metrics::record_put_object_quota_cache_decision("authoritative_mismatch_field", "quota_revision");
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONFIG_READ,
|
||||
config_read_started,
|
||||
);
|
||||
let lock_lost_check_started = rustfs_io_metrics::put_stage_timer();
|
||||
if metadata_guard.is_lock_lost() {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_config",
|
||||
@@ -918,6 +977,11 @@ pub(crate) async fn begin(
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_LOCK_LOST_CHECK,
|
||||
lock_lost_check_started,
|
||||
);
|
||||
let policy_evaluate_started = rustfs_io_metrics::put_stage_timer();
|
||||
if quota
|
||||
.as_ref()
|
||||
.is_some_and(|quota| quota.has_unsupported_reservation_protocol())
|
||||
@@ -926,6 +990,11 @@ pub(crate) async fn begin(
|
||||
return Err(StorageError::PartMissingOrCorrupt);
|
||||
}
|
||||
let durable_quota = quota.as_ref().filter(|quota| quota.uses_durable_reservations());
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_POLICY_EVALUATE,
|
||||
policy_evaluate_started,
|
||||
);
|
||||
let capability_proof_started = rustfs_io_metrics::put_stage_timer();
|
||||
let capability_proof = if durable_quota.is_some() {
|
||||
Some(
|
||||
crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
|
||||
@@ -934,7 +1003,12 @@ pub(crate) async fn begin(
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CAPABILITY_PROOF,
|
||||
capability_proof_started,
|
||||
);
|
||||
let durable_quota_limit = durable_quota.and_then(|quota| quota.quota);
|
||||
let snapshot_admission_started = rustfs_io_metrics::put_stage_timer();
|
||||
let reservation_protocol = durable_quota.and_then(|quota| quota.reservation_protocol);
|
||||
let snapshot_admission = match quota.as_ref().filter(|quota| !quota.uses_durable_reservations()) {
|
||||
Some(quota) => match (quota.quota, snapshot_admission) {
|
||||
@@ -954,6 +1028,10 @@ pub(crate) async fn begin(
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_SNAPSHOT_ADMISSION,
|
||||
snapshot_admission_started,
|
||||
);
|
||||
let legacy_data_movement = durable_quota_limit.is_none()
|
||||
&& quota.as_ref().and_then(|quota| quota.quota).is_some()
|
||||
&& snapshot_admission.is_none()
|
||||
@@ -965,11 +1043,21 @@ pub(crate) async fn begin(
|
||||
.then(|| quota.as_ref().and_then(|quota| quota.quota))
|
||||
.flatten()
|
||||
});
|
||||
let object_store_lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let store = if durable_quota_limit.is_some() {
|
||||
Some(metadata_sys::object_store_in(ctx).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_OBJECT_STORE_LOOKUP,
|
||||
object_store_lookup_started,
|
||||
);
|
||||
let context_build_started = rustfs_io_metrics::put_stage_timer();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_CONTEXT_BUILD,
|
||||
context_build_started,
|
||||
);
|
||||
Ok(QuotaContext {
|
||||
store,
|
||||
bucket: bucket.to_string(),
|
||||
@@ -988,6 +1076,15 @@ pub(crate) async fn begin(
|
||||
})
|
||||
}
|
||||
|
||||
fn quota_cache_reason(quota: Option<&crate::bucket::quota::BucketQuota>) -> &'static str {
|
||||
match quota {
|
||||
None => "no_quota",
|
||||
Some(quota) if quota.uses_durable_reservations() => "durable_quota",
|
||||
Some(quota) if quota.quota.is_some() => "snapshot_quota",
|
||||
Some(_) => "quota_without_limit",
|
||||
}
|
||||
}
|
||||
|
||||
fn quota_capability_error(bucket: &str, object: &str) -> StorageError {
|
||||
StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_capability",
|
||||
|
||||
@@ -65,9 +65,9 @@ use super::replication_target_boundary::{
|
||||
SsecPassthroughGate, TargetClient, VersionIdentityCapability, is_replication_target_offline_error,
|
||||
replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options,
|
||||
replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted,
|
||||
replication_put_object_header_size, replication_put_object_options, replication_target_head_is_newer_null_version,
|
||||
resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate, version_identity_capability_from_put,
|
||||
version_identity_drifted,
|
||||
replication_put_object_header_size, replication_put_object_options, replication_source_etag_is_comparable,
|
||||
replication_target_head_is_newer_null_version, resolve_read_api_version_id, ssec_passthrough_evidence_present,
|
||||
ssec_passthrough_gate, version_identity_capability_from_put, version_identity_drifted,
|
||||
};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
@@ -349,15 +349,22 @@ fn replica_etag_verification_enabled() -> bool {
|
||||
/// target that stores a transformed payload (e.g. undecoded `aws-chunked`
|
||||
/// frames, #6853) returns the ETag of what it actually wrote. Reporting
|
||||
/// COMPLETED over such a replica is silent corruption, so a decidable
|
||||
/// mismatch fails the replication instead. An SSE-C ciphertext passthrough
|
||||
/// transfer is exempt: the wire bytes are ciphertext while the source ETag is
|
||||
/// the plaintext MD5, and that path has its own HEAD-back audit.
|
||||
/// mismatch fails the replication instead. Encrypted sources are exempt: an
|
||||
/// SSE-C ciphertext passthrough sends ciphertext while the source ETag is the
|
||||
/// plaintext MD5 (that path has its own HEAD-back audit), and a managed-SSE
|
||||
/// (SSE-S3 / SSE-KMS) source stores a keyed fingerprint, or a legacy plaintext
|
||||
/// MD5, that the ETag of the target's re-encrypted replica can neither confirm
|
||||
/// nor refute. The checksum header on the replication PUT stays the integrity
|
||||
/// check for those objects.
|
||||
fn verify_single_part_replica(
|
||||
object_info: &ObjectInfo,
|
||||
response: &RemotePutObjectResponse,
|
||||
ciphertext_passthrough: bool,
|
||||
) -> std::result::Result<(), std::io::Error> {
|
||||
if ciphertext_passthrough || !replica_etag_verification_enabled() {
|
||||
if ciphertext_passthrough
|
||||
|| !replication_source_etag_is_comparable(&object_info.user_defined)
|
||||
|| !replica_etag_verification_enabled()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
if single_part_replica_etag_mismatch(object_info.etag.as_deref(), response.etag.as_deref()) {
|
||||
@@ -448,6 +455,26 @@ fn is_version_id_format_mismatch(err: &SdkError<HeadObjectError>) -> bool {
|
||||
is_version_id_mismatch(code, raw_status)
|
||||
}
|
||||
|
||||
/// A version-addressed HEAD rejected because the target cannot address the
|
||||
/// source version id (AWS S3: bodiless 400). Narrower than
|
||||
/// [`is_version_id_format_mismatch`]: a bodiless 403 may be a genuine
|
||||
/// `s3:GetObjectVersion` denial, so it is not accepted where a mutation or a
|
||||
/// completion verdict follows instead of another non-destructive HEAD.
|
||||
fn rejects_source_version_id(err: &SdkError<HeadObjectError>) -> bool {
|
||||
is_version_id_format_mismatch(err) && !has_raw_status(err, 403)
|
||||
}
|
||||
|
||||
/// Whether the target's latest version of `object` is a delete marker. For
|
||||
/// targets that mint their own version ids, which cannot be asked about the
|
||||
/// source marker id: S3 answers a versionless HEAD on such a key with 404
|
||||
/// (405 on some targets) plus `x-amz-delete-marker: true`.
|
||||
async fn latest_target_version_is_delete_marker(tgt_client: &TargetClient, object: &str) -> bool {
|
||||
match head_object_for_worker(tgt_client, &tgt_client.bucket, object, None).await {
|
||||
Ok(head) => head.delete_marker == Some(true),
|
||||
Err(e) => raw_header_value(e.as_ref(), "x-amz-delete-marker").as_deref() == Some("true"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn mark_replication_target_offline_if_needed(target_client: &Arc<TargetClient>, err: &(impl Display + ?Sized)) {
|
||||
if is_replication_target_offline_error(err) {
|
||||
ReplicationTargetStore::mark_target_offline(target_client).await;
|
||||
@@ -1846,7 +1873,17 @@ async fn verify_resync_head_result(
|
||||
// only absence proves completion. All other outcomes remain
|
||||
// failed (backlog#862 / #799 B13).
|
||||
let retryable = if roi.version_purge_status.is_empty() {
|
||||
matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Ambiguous)
|
||||
match classify_delete_marker_head_error(&err) {
|
||||
// A target that mints its own version ids never holds the
|
||||
// source marker id; verify the marker on the latest version.
|
||||
DeleteMarkerHeadOutcome::Ambiguous => {
|
||||
let mints_own = rejects_source_version_id(&err)
|
||||
|| ReplicationTargetStore::version_identity_capability(&target_client.arn)
|
||||
.version_addressing_unreliable();
|
||||
!(mints_own && latest_target_version_is_delete_marker(target_client, &roi.name).await)
|
||||
}
|
||||
DeleteMarkerHeadOutcome::Exists | DeleteMarkerHeadOutcome::Missing => false,
|
||||
}
|
||||
} else {
|
||||
!matches!(classify_delete_marker_head_error(&err), DeleteMarkerHeadOutcome::Missing)
|
||||
};
|
||||
@@ -3567,7 +3604,16 @@ async fn replicate_delete_to_target(
|
||||
}
|
||||
}
|
||||
|
||||
if !is_version_purge && dobj.delete_object.delete_marker && dobj.delete_object.delete_marker_version_id.is_some() {
|
||||
// A target that mints its own version ids never holds the source marker
|
||||
// id, so a version-addressed pre-check cannot succeed there; the marker
|
||||
// is created by the versionless DELETE below.
|
||||
let version_addressable =
|
||||
!ReplicationTargetStore::version_identity_capability(&tgt_client.arn).version_addressing_unreliable();
|
||||
if version_addressable
|
||||
&& !is_version_purge
|
||||
&& dobj.delete_object.delete_marker
|
||||
&& dobj.delete_object.delete_marker_version_id.is_some()
|
||||
{
|
||||
match head_object_for_worker(
|
||||
tgt_client.as_ref(),
|
||||
&tgt_client.bucket,
|
||||
@@ -3576,6 +3622,11 @@ async fn replicate_delete_to_target(
|
||||
)
|
||||
.await
|
||||
{
|
||||
// The verdict is not known yet (e.g. no PUT since restart) and the
|
||||
// target rejected the source id: learn it as the PUT path does.
|
||||
Err(e) if rejects_source_version_id(e.as_ref()) => {
|
||||
ReplicationTargetStore::record_version_identity_capability(&tgt_client.arn, VersionIdentityCapability::MintsOwn);
|
||||
}
|
||||
Ok(head) if head.delete_marker == Some(true) => {
|
||||
// Some targets expose an existing marker as a successful HEAD
|
||||
// with the marker bit instead of the usual 405. Treat that as
|
||||
@@ -5826,6 +5877,57 @@ mod tests {
|
||||
use super::super::replication_filemeta_boundary::ReplicateTargetDecision;
|
||||
use super::super::replication_object_decision_boundary::ReplicationMultipartPlanError;
|
||||
|
||||
#[test]
|
||||
fn verify_single_part_replica_exempts_managed_sse_sources() {
|
||||
// A new encrypted write stores a keyed fingerprint; a target that
|
||||
// computes its own ETag answers with the MD5 of the plaintext it
|
||||
// received. Both are 32 hex characters, so without the exemption the
|
||||
// audit would fail every SSE-S3 / SSE-KMS replica (rustfs/backlog).
|
||||
let fingerprint = "00112233445566778899aabbccddeeff";
|
||||
let replica_md5 = "ffeeddccbbaa99887766554433221100";
|
||||
let response = RemotePutObjectResponse {
|
||||
version_id: None,
|
||||
etag: Some(replica_md5.to_string()),
|
||||
};
|
||||
for sse in ["AES256", "aws:kms"] {
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "encrypted".to_string(),
|
||||
etag: Some(fingerprint.to_string()),
|
||||
user_defined: HashMap::from([("x-amz-server-side-encryption".to_string(), sse.to_string())]).into(),
|
||||
..Default::default()
|
||||
};
|
||||
verify_single_part_replica(&object_info, &response, false)
|
||||
.unwrap_or_else(|err| panic!("{sse}: a managed-SSE source must not be audited against the replica MD5: {err}"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_single_part_replica_still_fails_plaintext_mismatch() {
|
||||
if !replica_etag_verification_enabled() {
|
||||
return;
|
||||
}
|
||||
let object_info = ObjectInfo {
|
||||
bucket: "source".to_string(),
|
||||
name: "plain".to_string(),
|
||||
etag: Some("00112233445566778899aabbccddeeff".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let mismatch = RemotePutObjectResponse {
|
||||
version_id: None,
|
||||
etag: Some("ffeeddccbbaa99887766554433221100".to_string()),
|
||||
};
|
||||
let err = verify_single_part_replica(&object_info, &mismatch, false)
|
||||
.expect_err("a plaintext source whose replica MD5 differs must fail the replication");
|
||||
assert_eq!(err.to_string(), REPLICA_ETAG_MISMATCH_ERROR);
|
||||
|
||||
let matching = RemotePutObjectResponse {
|
||||
version_id: None,
|
||||
etag: object_info.etag.clone(),
|
||||
};
|
||||
verify_single_part_replica(&object_info, &matching, false).expect("matching plaintext MD5s pass the audit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multipart_read_plan_preserves_legacy_plain_part_ranges() {
|
||||
const MIB: usize = 1024 * 1024;
|
||||
@@ -6675,6 +6777,93 @@ mod tests {
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
}
|
||||
|
||||
/// AWS S3 answers a HEAD for a version id it did not mint with a bodiless
|
||||
/// 400. On a target with no verdict yet, that teaches `MintsOwn` and the
|
||||
/// marker is still created with one versionless DELETE.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_learns_mints_own_from_bodiless_400() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(2, |line| {
|
||||
if line.starts_with("HEAD ") {
|
||||
empty_response("400 Bad Request")
|
||||
} else if line.starts_with("DELETE ") {
|
||||
empty_response("204 No Content")
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
|
||||
assert_eq!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::MintsOwn
|
||||
);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 2, "{seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
assert!(seen[1].starts_with("DELETE "), "{seen:?}");
|
||||
assert!(!seen[1].contains("versionId="), "marker creation must remain versionless: {seen:?}");
|
||||
}
|
||||
|
||||
/// A bodiless 403 on the version-addressed HEAD may be a genuine
|
||||
/// GetObjectVersion denial on an id-adopting target. It must fail closed:
|
||||
/// no DELETE, and no `MintsOwn` verdict cached for the ARN.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_keeps_bodiless_403_failure_closed() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |line| {
|
||||
if line.starts_with("HEAD ") {
|
||||
empty_response("403 Forbidden")
|
||||
} else if line.starts_with("DELETE ") {
|
||||
empty_response("204 No Content")
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed, "{rinfo:?}");
|
||||
assert_ne!(
|
||||
ReplicationTargetStore::version_identity_capability(&target.arn),
|
||||
VersionIdentityCapability::MintsOwn
|
||||
);
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "a denied HEAD must not trigger DELETE: {seen:?}");
|
||||
assert!(seen[0].starts_with("HEAD "), "{seen:?}");
|
||||
}
|
||||
|
||||
/// A target already known to mint its own version ids is never asked for
|
||||
/// the source marker id: the marker is created by the versionless DELETE.
|
||||
#[tokio::test]
|
||||
async fn live_delete_marker_worker_skips_version_head_on_mints_own_target() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |line| {
|
||||
if line.starts_with("DELETE ") {
|
||||
empty_response("204 No Content")
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
register_test_target(&target).await;
|
||||
ReplicationTargetStore::record_version_identity_capability(&target.arn, VersionIdentityCapability::MintsOwn);
|
||||
let dobj = marker_creation_dobj(&target.arn);
|
||||
|
||||
let rinfo = replicate_delete_to_target(&dobj, target.clone(), None).await;
|
||||
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Completed, "{rinfo:?}");
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "no version-addressed HEAD on a MintsOwn target: {seen:?}");
|
||||
assert!(seen[0].starts_with("DELETE "), "{seen:?}");
|
||||
assert!(!seen[0].contains("versionId="), "marker creation must remain versionless: {seen:?}");
|
||||
}
|
||||
|
||||
/// A target answers `HEAD ?versionId=<delete marker>` with a bodiless
|
||||
/// 405, so the SDK reports no error code. That is the marker having
|
||||
/// propagated, not a failure (rustfs/backlog#2479).
|
||||
@@ -6765,6 +6954,76 @@ mod tests {
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
/// AWS answers `HEAD ?versionId=<source marker id>` with a bodiless 400.
|
||||
/// The marker is verified through the latest version instead: a
|
||||
/// versionless 404 with `x-amz-delete-marker: true` proves it propagated.
|
||||
#[tokio::test]
|
||||
async fn resync_verification_accepts_latest_delete_marker_when_target_rejects_source_id() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(2, |line| {
|
||||
if line.contains("versionId=") {
|
||||
empty_response("400 Bad Request")
|
||||
} else {
|
||||
"HTTP/1.1 404 Not Found\r\nX-Amz-Delete-Marker: true\r\nX-Amz-Version-Id: target-marker\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
.to_string()
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
let roi = delete_marker_roi();
|
||||
let mut st = TargetReplicationResyncStatus::default();
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_none(), "{err:?}");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 1, 0));
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert!(!seen[1].contains("versionId="), "the re-check must address the latest version: {seen:?}");
|
||||
}
|
||||
|
||||
/// A live latest version on a target that rejects the source id means the
|
||||
/// marker has not propagated: the verification stays failed.
|
||||
#[tokio::test]
|
||||
async fn resync_verification_fails_delete_marker_when_latest_target_version_is_live() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(2, |line| {
|
||||
if line.contains("versionId=") {
|
||||
empty_response("400 Bad Request")
|
||||
} else {
|
||||
empty_response("200 OK")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
let roi = delete_marker_roi();
|
||||
let mut st = TargetReplicationResyncStatus::default();
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_some(), "a live latest version must not verify the marker");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 0, 1));
|
||||
server.join().expect("test HTTP server should finish");
|
||||
}
|
||||
|
||||
/// A bodiless 403 may be a genuine permission denial: it is not evidence
|
||||
/// that the target mints its own ids, so no latest-version re-check runs.
|
||||
#[tokio::test]
|
||||
async fn resync_verification_keeps_delete_marker_failed_on_bodiless_403() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(1, |_| empty_response("403 Forbidden"));
|
||||
let target = test_target_client(endpoint);
|
||||
let roi = delete_marker_roi();
|
||||
let mut st = TargetReplicationResyncStatus::default();
|
||||
|
||||
let head_result =
|
||||
head_object_for_worker(target.as_ref(), &target.bucket, &roi.name, roi.version_id.map(|v| v.to_string())).await;
|
||||
let (size, err) = verify_resync_head_result(head_result, &roi, &mut st, &target).await;
|
||||
|
||||
assert!(err.is_some(), "a 403 must not count as a propagated delete marker");
|
||||
assert_eq!((size, st.replicated_count, st.failed_count), (0, 0, 1));
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 1, "{seen:?}");
|
||||
}
|
||||
|
||||
/// Serves exactly `requests` connections, answering each from
|
||||
/// `respond(request_line)`, and returns the request lines it saw. Reads
|
||||
/// the whole request (headers plus `Content-Length` body, honoring
|
||||
|
||||
@@ -157,6 +157,19 @@ pub(crate) fn replication_object_is_ssec_encrypted(user_defined: &HashMap<String
|
||||
rustfs_replication::is_ssec_encrypted(user_defined)
|
||||
}
|
||||
|
||||
/// Whether the stored ETag of a replication source can be compared with the
|
||||
/// ETag a target returns for the bytes it received. Only an unencrypted source
|
||||
/// stores the MD5 of the bytes that go on the wire. A managed-SSE (SSE-S3 /
|
||||
/// SSE-KMS) or SSE-C source stores a keyed fingerprint (new writes) or a legacy
|
||||
/// plaintext MD5, and the target re-encrypts what it receives, so neither
|
||||
/// equality nor inequality of the two ETags proves anything about the replica.
|
||||
pub(crate) fn replication_source_etag_is_comparable(user_defined: &HashMap<String, String>) -> bool {
|
||||
matches!(
|
||||
classify_replication_source_encryption(user_defined),
|
||||
ReplicationSourceEncryption::Plaintext
|
||||
)
|
||||
}
|
||||
|
||||
/// HeadObjectOutput adapter over the pure SSE-C passthrough evidence
|
||||
/// judgment owned by `rustfs-replication`: extract the echoed
|
||||
/// customer-algorithm header and let the crate-owned policy decide.
|
||||
@@ -578,6 +591,22 @@ mod tests {
|
||||
use time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn replication_source_etag_is_comparable_only_for_plaintext_sources() {
|
||||
assert!(replication_source_etag_is_comparable(&HashMap::new()));
|
||||
for (key, value) in [
|
||||
(metadata_keys::SERVER_SIDE_ENCRYPTION, "AES256"),
|
||||
(metadata_keys::SERVER_SIDE_ENCRYPTION, "aws:kms"),
|
||||
(SSEC_ALGORITHM_HEADER, "AES256"),
|
||||
] {
|
||||
let metadata = HashMap::from([(key.to_string(), value.to_string())]);
|
||||
assert!(
|
||||
!replication_source_etag_is_comparable(&metadata),
|
||||
"{key}={value} must not expose its stored ETag to the replica audit"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// backlog#1735 A3b SSE key-case finding. Every RustFS writer persists the
|
||||
/// SSE intent as lowercase `x-amz-server-side-encryption`
|
||||
/// (`encryption_material_to_metadata`; header-derived keys come from a
|
||||
|
||||
@@ -353,6 +353,13 @@ fn apply_peer_replay_response(
|
||||
}
|
||||
}
|
||||
|
||||
fn rename_data_grpc_stage(path: &str) -> bool {
|
||||
matches!(
|
||||
path,
|
||||
"/node_service.NodeService/RenameData" | "/node_service.NodeService/RenameDataAtIncarnation"
|
||||
)
|
||||
}
|
||||
|
||||
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
|
||||
where
|
||||
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
|
||||
@@ -370,6 +377,12 @@ where
|
||||
}
|
||||
|
||||
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
|
||||
let observe_rename_data = rustfs_io_metrics::put_stage_metrics_enabled() && rename_data_grpc_stage(request.uri().path());
|
||||
let request_scope_started = if observe_rename_data {
|
||||
rustfs_io_metrics::put_stage_timer()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let authenticated = self.audience.as_ref().is_some_and(|_| {
|
||||
request
|
||||
.headers()
|
||||
@@ -404,11 +417,30 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REQUEST_SCOPE,
|
||||
request_scope_started,
|
||||
);
|
||||
|
||||
let audience = self.audience.clone();
|
||||
let future = self.inner.call(request);
|
||||
Box::pin(async move {
|
||||
let response = future.await?;
|
||||
let response = if observe_rename_data {
|
||||
rustfs_io_metrics::observe_put_stage_future(
|
||||
future,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_POLL_PENDING_COUNT,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_TRANSPORT_CALL_FIRST_PENDING_TO_READY,
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
future.await?
|
||||
};
|
||||
let replay_response_started = if observe_rename_data {
|
||||
rustfs_io_metrics::put_stage_timer()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let (Some(audience), Some(challenge)) = (audience, challenge) {
|
||||
let response_state = verify_tonic_peer_replay_capabilities_response(&audience, challenge, response.headers());
|
||||
if let Err(error) = &response_state
|
||||
@@ -426,6 +458,10 @@ where
|
||||
}
|
||||
apply_peer_replay_response(audience, sent_state, response_state);
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_REPLAY_RESPONSE,
|
||||
replay_response_started,
|
||||
);
|
||||
Ok(response)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -325,6 +325,24 @@ fn map_status(status: tonic::Status) -> NetworkPeerProbeError {
|
||||
match status.code() {
|
||||
Code::Unavailable | Code::Unknown => NetworkPeerProbeError::Unreachable,
|
||||
Code::DeadlineExceeded => NetworkPeerProbeError::TimedOut,
|
||||
Code::Cancelled | Code::Internal => {
|
||||
let mut source = std::error::Error::source(&status);
|
||||
while let Some(error) = source {
|
||||
if error
|
||||
.downcast_ref::<h2::Error>()
|
||||
.is_some_and(|error| matches!(error.reason(), Some(h2::Reason::CANCEL | h2::Reason::NO_ERROR)))
|
||||
{
|
||||
return NetworkPeerProbeError::Unreachable;
|
||||
}
|
||||
source = if let Some(error) = error.downcast_ref::<std::io::Error>() {
|
||||
// io::Error::source skips its immediate custom error.
|
||||
error.get_ref().map(|inner| inner as &(dyn std::error::Error + 'static))
|
||||
} else {
|
||||
error.source()
|
||||
};
|
||||
}
|
||||
NetworkPeerProbeError::ProtocolFailure
|
||||
}
|
||||
_ => NetworkPeerProbeError::ProtocolFailure,
|
||||
}
|
||||
}
|
||||
@@ -335,6 +353,35 @@ mod tests {
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints};
|
||||
|
||||
#[test]
|
||||
fn peer_transport_resets_are_unreachable_without_reclassifying_application_errors() {
|
||||
for reason in [h2::Reason::CANCEL, h2::Reason::NO_ERROR] {
|
||||
let status = tonic::Status::from_error(Box::new(h2::Error::from(reason)));
|
||||
assert!(matches!(status.code(), Code::Cancelled | Code::Internal));
|
||||
let code = status.code();
|
||||
assert_eq!(map_status(status), NetworkPeerProbeError::Unreachable);
|
||||
let mut wrapped = tonic::Status::new(code, "wrapped transport reset");
|
||||
wrapped.set_source(Arc::new(std::io::Error::other(h2::Error::from(reason))));
|
||||
assert_eq!(map_status(wrapped), NetworkPeerProbeError::Unreachable);
|
||||
}
|
||||
for reason in [h2::Reason::PROTOCOL_ERROR, h2::Reason::COMPRESSION_ERROR] {
|
||||
let status = tonic::Status::from_error(Box::new(h2::Error::from(reason)));
|
||||
assert_eq!(map_status(status), NetworkPeerProbeError::ProtocolFailure);
|
||||
}
|
||||
for status in [
|
||||
tonic::Status::cancelled("peer cancelled the application request"),
|
||||
tonic::Status::internal("peer rejected the probe"),
|
||||
tonic::Status::unauthenticated("authentication rejected"),
|
||||
tonic::Status::permission_denied("authorization rejected"),
|
||||
] {
|
||||
assert_eq!(map_status(status), NetworkPeerProbeError::ProtocolFailure);
|
||||
}
|
||||
assert_eq!(
|
||||
map_status(tonic::Status::deadline_exceeded("probe deadline elapsed")),
|
||||
NetworkPeerProbeError::TimedOut
|
||||
);
|
||||
}
|
||||
|
||||
fn pacing(duration: Duration) -> Arc<DiagnosticPacing> {
|
||||
NetworkPeerProbeClient::from_endpoint_pools(&topology())
|
||||
.with_diagnostic_pacing(Instant::now(), duration)
|
||||
|
||||
@@ -2279,6 +2279,7 @@ impl RemoteDisk {
|
||||
self.execute_with_timeout_for_op(
|
||||
"rename_data",
|
||||
|| async {
|
||||
let prepare_started = rustfs_io_metrics::put_stage_timer();
|
||||
let file_info = compat_json(fi)?;
|
||||
let file_info_bin = encode_file_info_msgpack(fi)?;
|
||||
let mut client = self.get_client().await?;
|
||||
@@ -2306,24 +2307,67 @@ impl RemoteDisk {
|
||||
} else {
|
||||
attach_mutation_body_digest(&mut request, canonical_body, "rename_data")?;
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_PREPARE,
|
||||
prepare_started,
|
||||
);
|
||||
|
||||
let response = if incarnation_bound {
|
||||
// Older peers return Unimplemented before mutation; never downgrade.
|
||||
client.rename_data_at_incarnation(request).await?
|
||||
} else {
|
||||
client.rename_data(request).await?
|
||||
}
|
||||
.into_inner();
|
||||
let rpc_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = rustfs_io_metrics::observe_put_stage_future(
|
||||
async {
|
||||
if incarnation_bound {
|
||||
// Older peers return Unimplemented before mutation; never downgrade.
|
||||
client.rename_data_at_incarnation(request).await
|
||||
} else {
|
||||
client.rename_data(request).await
|
||||
}
|
||||
},
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_POLL_PENDING_COUNT,
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC_AWAIT_FIRST_PENDING_TO_READY,
|
||||
)
|
||||
.await?;
|
||||
let response_process_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response_headers_started = rustfs_io_metrics::put_stage_timer();
|
||||
let _ = response.metadata().len();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_HEADERS,
|
||||
response_headers_started,
|
||||
);
|
||||
let into_inner_started = rustfs_io_metrics::put_stage_timer();
|
||||
let response = response.into_inner();
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_INTO_INNER,
|
||||
into_inner_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RPC,
|
||||
rpc_started,
|
||||
);
|
||||
|
||||
let success_check_started = rustfs_io_metrics::put_stage_timer();
|
||||
if !response.success {
|
||||
return Err(response.error.unwrap_or_default().into());
|
||||
}
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_SUCCESS_CHECK,
|
||||
success_check_started,
|
||||
);
|
||||
|
||||
let decode_started = rustfs_io_metrics::put_stage_timer();
|
||||
let rename_data_resp = decode_msgpack_or_json::<RenameDataResp>(
|
||||
&response.rename_data_resp_bin,
|
||||
&response.rename_data_resp,
|
||||
"RenameDataResp",
|
||||
)?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_DECODE,
|
||||
decode_started,
|
||||
);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_REMOTE_CLIENT_RESPONSE_PROCESS,
|
||||
response_process_started,
|
||||
);
|
||||
|
||||
Ok(rename_data_resp)
|
||||
},
|
||||
@@ -5116,6 +5160,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![0x14, 0x35]),
|
||||
old_current_size: Some(crate::disk::OldCurrentSize::Present(64 * 1024)),
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
let json = serde_json::to_string(&response).expect("legacy rename_data JSON response should encode");
|
||||
let decode_before = rustfs_io_metrics::internode_metrics::global_internode_metrics().msgpack_json_decode_total_for_test();
|
||||
@@ -5518,6 +5564,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![1_u8; 32]),
|
||||
old_current_size: Some(crate::disk::OldCurrentSize::Present(4096)),
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
let json = serde_json::to_vec(&response).expect("rename data response json should encode");
|
||||
let named_msgpack = rmp_serde::encode::to_vec_named(&response).expect("rename data response named msgpack should encode");
|
||||
|
||||
@@ -3691,6 +3691,55 @@ fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
|
||||
}
|
||||
}
|
||||
|
||||
fn rebalance_worker_read_lock_error(err: rustfs_lock::LockError, object: &str) -> Error {
|
||||
match err {
|
||||
rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "read",
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
object: object.to_string(),
|
||||
required,
|
||||
achieved,
|
||||
},
|
||||
other => Error::Lock(other),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read-only admission to an already durable rebalance run. Keeping this
|
||||
/// separate from the activation writer prevents admission from authorizing a
|
||||
/// pool or rebalance metadata write under shared namespace locks.
|
||||
pub(crate) struct PoolRebalanceWorkerFence {
|
||||
pool_meta_guard: rustfs_lock::NamespaceLockGuard,
|
||||
rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
}
|
||||
|
||||
impl PoolRebalanceWorkerFence {
|
||||
pub(crate) fn set_fleet_proof(
|
||||
&mut self,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
) {
|
||||
self.fleet_proof = fleet_proof;
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_held(&self) -> Result<()> {
|
||||
if self.pool_meta_guard.is_lock_lost()
|
||||
|| self.pool_meta_guard.is_released()
|
||||
|| self.rebalance_meta_guard.is_lock_lost()
|
||||
|| self.rebalance_meta_guard.is_released()
|
||||
{
|
||||
return Err(Error::other("worker admission lock lost before rebalance activation"));
|
||||
}
|
||||
if self
|
||||
.fleet_proof
|
||||
.as_ref()
|
||||
.is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
|
||||
{
|
||||
return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct PoolRebalanceActivationFence {
|
||||
pool_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
rebalance_meta_guard: Arc<rustfs_lock::NamespaceLockGuard>,
|
||||
@@ -3762,6 +3811,35 @@ tokio::task_local! {
|
||||
pub(crate) static REBALANCE_ACTIVATION_LOCK_ATTEMPT: Arc<tokio::sync::Notify>;
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_pool_rebalance_worker_locks<S>(pool: Arc<S>) -> Result<PoolRebalanceWorkerFence>
|
||||
where
|
||||
S: crate::storage_api_contracts::namespace::NamespaceLocking<
|
||||
Error = Error,
|
||||
NamespaceLock = rustfs_lock::NamespaceLockWrapper,
|
||||
>,
|
||||
{
|
||||
// Match activation and entry admission: rebalance.bin -> pool.bin. Joining
|
||||
// the same durable run must coexist with entries retaining their run fence.
|
||||
let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
|
||||
#[cfg(test)]
|
||||
let _ = REBALANCE_ACTIVATION_LOCK_ATTEMPT.try_with(|attempted| attempted.notify_one());
|
||||
let rebalance_meta_guard = rebalance_meta_lock
|
||||
.get_read_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| rebalance_worker_read_lock_error(err, REBAL_META_NAME))?;
|
||||
let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
|
||||
let pool_meta_guard = pool_meta_lock
|
||||
.get_read_lock(get_lock_acquire_timeout())
|
||||
.await
|
||||
.map_err(|err| rebalance_worker_read_lock_error(err, POOL_META_NAME))?;
|
||||
|
||||
Ok(PoolRebalanceWorkerFence {
|
||||
pool_meta_guard,
|
||||
rebalance_meta_guard,
|
||||
fleet_proof: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
|
||||
pool: Arc<S>,
|
||||
fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
|
||||
@@ -12430,6 +12508,18 @@ impl ECStore {
|
||||
Ok(pool_meta)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_runtime_pool_meta_under_worker_fence(
|
||||
&self,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
worker_fence: &PoolRebalanceWorkerFence,
|
||||
operation: &str,
|
||||
) -> Result<PoolMeta> {
|
||||
worker_fence.ensure_held()?;
|
||||
let pool_meta = self.load_runtime_pool_meta_observing(write_state, operation).await?;
|
||||
worker_fence.ensure_held()?;
|
||||
Ok(pool_meta)
|
||||
}
|
||||
|
||||
pub(crate) async fn load_runtime_pool_meta_under_activation_fence(
|
||||
&self,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
|
||||
@@ -44,7 +44,11 @@ use std::{
|
||||
use time::OffsetDateTime;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use tracing::warn;
|
||||
use tracing::{error, trace, warn};
|
||||
|
||||
const EVENT_SERVER_PING: &str = "server_ping";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_DIAGNOSTICS: &str = "diagnostics";
|
||||
|
||||
use shadow_rs::shadow;
|
||||
|
||||
@@ -159,11 +163,22 @@ async fn is_server_resolvable(endpoint: &Endpoint) -> Result<()> {
|
||||
|
||||
let response: PingResponse = client.ping(request).await?.into_inner();
|
||||
|
||||
let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
|
||||
if let Err(e) = ping_response_body {
|
||||
eprintln!("{e}");
|
||||
if flatbuffers::root::<PingBody>(&response.body).is_err() {
|
||||
error!(
|
||||
event = EVENT_SERVER_PING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DIAGNOSTICS,
|
||||
result = "invalid_response",
|
||||
"Server ping response decoding failed"
|
||||
);
|
||||
} else {
|
||||
println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
|
||||
trace!(
|
||||
event = EVENT_SERVER_PING,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DIAGNOSTICS,
|
||||
result = "success",
|
||||
"Server ping completed"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::cleanup_runtime;
|
||||
use crate::disk::io_admission::{
|
||||
AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context,
|
||||
with_disk_io_context, with_disk_io_permit,
|
||||
@@ -484,10 +485,10 @@ impl LocalDiskWrapper {
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
self.track_cleanup_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { Box::pin(self.disk.delete_with_namespace_owner(volume, path, opts, namespace_owner)).await },
|
||||
|| async { Box::pin(self.cleanup_delete(volume, path, opts, namespace_owner)).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
@@ -1475,6 +1476,28 @@ impl LocalDiskWrapper {
|
||||
)
|
||||
}
|
||||
|
||||
async fn cleanup_delete(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
options: DeleteOptions,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
if !cleanup_runtime::enabled()? {
|
||||
return self.disk.delete_with_namespace_owner(volume, path, options, owner).await;
|
||||
}
|
||||
let disk = self.disk.clone();
|
||||
let volume = volume.to_owned();
|
||||
let path = path.to_owned();
|
||||
let guarded = owner.is_some();
|
||||
let operation = async move { disk.delete_with_namespace_owner(&volume, &path, options, owner).await };
|
||||
if guarded {
|
||||
cleanup_runtime::guarded_disk_operation(operation).await
|
||||
} else {
|
||||
cleanup_runtime::disk_operation(operation).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a delete under an owned coordinator task when a publication guard
|
||||
/// is present. This keeps the guard alive if the RPC waiter is cancelled
|
||||
/// while the local namespace mutation is still in progress.
|
||||
@@ -1493,12 +1516,13 @@ impl LocalDiskWrapper {
|
||||
} else {
|
||||
get_max_timeout_duration()
|
||||
};
|
||||
let cleanup_owner = external_guard.clone();
|
||||
run_owned_mutation(external_guard, move || async move {
|
||||
operation
|
||||
.track_disk_health_mutation(
|
||||
.track_cleanup_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { operation.disk.delete(&volume, &path, options).await },
|
||||
|| async { operation.cleanup_delete(&volume, &path, options, cleanup_owner).await },
|
||||
timeout_duration,
|
||||
)
|
||||
.await
|
||||
@@ -2092,6 +2116,31 @@ impl LocalDiskWrapper {
|
||||
self.track_disk_health_with_op("unknown", operation, timeout_duration).await
|
||||
}
|
||||
|
||||
/// Cleanup latency includes waiting behind a node-wide execution budget,
|
||||
/// so a waiter deadline alone is not evidence that this disk is faulty.
|
||||
/// The active disk health probe retains its original timeout policy.
|
||||
async fn track_cleanup_mutation<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
mutation: DiskMetricMutation,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
// Runtime initialization must not prevent control-plane release of
|
||||
// reader tokens. Disk I/O entries still propagate configuration errors.
|
||||
let action = if cleanup_runtime::enabled().unwrap_or(false) {
|
||||
TimeoutHealthAction::IgnoreFailure
|
||||
} else {
|
||||
TimeoutHealthAction::MarkFailure
|
||||
};
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation(op, operation, timeout_duration, action, mutation)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn track_disk_health_mutation<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
@@ -2589,9 +2638,37 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn release_snapshot_lease(&self, volume: &str, path: &str, token: SnapshotLeaseToken) -> Result<()> {
|
||||
self.track_disk_health_with_op(
|
||||
// Quota fence release is control-plane work and may wait for in-flight
|
||||
// mutations. Keep its original waiter lifecycle instead of creating
|
||||
// detached cleanup tasks for repeated fence-release RPCs.
|
||||
if volume == super::RUSTFS_META_BUCKET && super::is_quota_mutation_fence_path(path) {
|
||||
return self
|
||||
.track_disk_health_with_op(
|
||||
"release_snapshot_lease",
|
||||
|| async { self.disk.release_snapshot_lease(volume, path, token).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
self.track_cleanup_mutation(
|
||||
"release_snapshot_lease",
|
||||
|| async { self.disk.release_snapshot_lease(volume, path, token).await },
|
||||
DiskMetricMutation::None,
|
||||
|| async {
|
||||
if self.disk.release_snapshot_lease_without_cleanup(volume, path, token).await {
|
||||
return Ok(());
|
||||
}
|
||||
if !cleanup_runtime::enabled().unwrap_or(false) {
|
||||
// The raw release removes the token before trying the
|
||||
// reclaim budget. Initialization failure therefore keeps
|
||||
// the cleanup intent and data, without leaking a reader
|
||||
// or falling back to unbudgeted filesystem deletion.
|
||||
return self.disk.release_snapshot_lease(volume, path, token).await;
|
||||
}
|
||||
let disk = self.disk.clone();
|
||||
let volume = volume.to_owned();
|
||||
let path = path.to_owned();
|
||||
cleanup_runtime::release_lease(async move { disk.release_snapshot_lease(&volume, &path, token).await }).await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
@@ -2616,10 +2693,22 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
} else {
|
||||
get_max_timeout_duration()
|
||||
};
|
||||
self.track_disk_health_mutation(
|
||||
self.track_cleanup_mutation(
|
||||
"delete_data_dir",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { self.disk.delete_data_dir(volume, path, opts).await },
|
||||
|| async {
|
||||
if !cleanup_runtime::enabled()? {
|
||||
return self.disk.delete_data_dir(volume, path, opts).await;
|
||||
}
|
||||
let disk = self.disk.clone();
|
||||
let volume = volume.to_owned();
|
||||
let path = path.to_owned();
|
||||
cleanup_runtime::disk_operation(async move {
|
||||
let _scope = scope;
|
||||
disk.delete_data_dir(&volume, &path, opts).await
|
||||
})
|
||||
.await
|
||||
},
|
||||
timeout,
|
||||
)
|
||||
.await
|
||||
@@ -2866,10 +2955,10 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
self.track_cleanup_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { self.disk.delete(volume, path, opt).await },
|
||||
|| async { self.cleanup_delete(volume, path, opt, None).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
@@ -2902,11 +2991,25 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn write_all(&self, volume: &str, path: &str, data: Bytes) -> Result<()> {
|
||||
self.track_disk_health_mutation(
|
||||
let cleanup = path.rsplit('/').next() == Some(cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE);
|
||||
self.track_disk_health_with_op_timeout_action_and_mutation(
|
||||
"write_all",
|
||||
DiskMetricMutation::Write,
|
||||
|| async { self.disk.write_all(volume, path, data).await },
|
||||
|| async {
|
||||
if !cleanup || !cleanup_runtime::enabled()? {
|
||||
return self.disk.write_all(volume, path, data).await;
|
||||
}
|
||||
let disk = self.disk.clone();
|
||||
let volume = volume.to_owned();
|
||||
let path = path.to_owned();
|
||||
cleanup_runtime::disk_operation(async move { disk.write_all(&volume, &path, data).await }).await
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
if cleanup && cleanup_runtime::enabled()? {
|
||||
TimeoutHealthAction::IgnoreFailure
|
||||
} else {
|
||||
TimeoutHealthAction::MarkFailure
|
||||
},
|
||||
DiskMetricMutation::Write,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -3299,6 +3402,256 @@ mod tests {
|
||||
let _ = walk.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalid_cleanup_config_does_not_leak_snapshot_readers() {
|
||||
const CHILD: &str = "RUSTFS_CLEANUP_INVALID_LEASE_TEST_CHILD";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
let output = tokio::task::spawn_blocking(|| {
|
||||
std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args([
|
||||
"--exact",
|
||||
"disk::disk_store::tests::invalid_cleanup_config_does_not_leak_snapshot_readers",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, "1")
|
||||
.env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true")
|
||||
.env("RUSTFS_CLEANUP_ASYNC_THREADS", "0")
|
||||
.output()
|
||||
.expect("run with invalid cleanup configuration")
|
||||
})
|
||||
.await
|
||||
.expect("child process task");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().expect("temporary disk");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 path")).expect("endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(disk.clone(), false);
|
||||
let volume = "invalid-cleanup-config";
|
||||
let path = "object/data";
|
||||
wrapper.make_volume(volume).await.expect("bucket");
|
||||
wrapper
|
||||
.write_all(volume, "object/data/part.1", Bytes::from_static(b"live shard"))
|
||||
.await
|
||||
.expect("write shard");
|
||||
let token = wrapper.acquire_snapshot_lease(volume, path).await.expect("GET lease");
|
||||
wrapper
|
||||
.release_snapshot_lease(volume, path, token)
|
||||
.await
|
||||
.expect("control-only release must not depend on cleanup initialization");
|
||||
assert!(matches!(
|
||||
wrapper.renew_snapshot_lease(volume, path, token).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
));
|
||||
|
||||
let token = wrapper
|
||||
.acquire_snapshot_lease(volume, path)
|
||||
.await
|
||||
.expect("reader of old data");
|
||||
// Register reclaim intent directly; this only updates the lease
|
||||
// registry while a reader is present and performs no filesystem I/O.
|
||||
assert_eq!(
|
||||
disk.delete_data_dir(
|
||||
volume,
|
||||
path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.expect("defer reclaim"),
|
||||
DataDirDeleteStatus::Deferred
|
||||
);
|
||||
assert!(
|
||||
wrapper.release_snapshot_lease(volume, path, token).await.is_err(),
|
||||
"invalid configuration must still reject reclamation"
|
||||
);
|
||||
assert!(
|
||||
matches!(wrapper.renew_snapshot_lease(volume, path, token).await, Err(DiskError::FileNotFound)),
|
||||
"failed reclamation must not strand a reader token"
|
||||
);
|
||||
assert_eq!(
|
||||
wrapper.read_all(volume, "object/data/part.1").await.expect("retained data"),
|
||||
Bytes::from_static(b"live shard"),
|
||||
"configuration failure must not fall back to unbudgeted deletion"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rpc_cleanup_entries_share_disk_budget() {
|
||||
const CHILD: &str = "RUSTFS_CLEANUP_RPC_TEST_CHILD";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
let output = tokio::task::spawn_blocking(|| {
|
||||
std::process::Command::new(std::env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
"disk::disk_store::tests::rpc_cleanup_entries_share_disk_budget",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, "1")
|
||||
.env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true")
|
||||
.env("RUSTFS_CLEANUP_ASYNC_THREADS", "1")
|
||||
.env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "1")
|
||||
.env("RUSTFS_CLEANUP_DISK_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_GC_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_CPUS", "none")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP")
|
||||
.output()
|
||||
.unwrap()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.unwrap());
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let volume = "cleanup-receiver";
|
||||
let path = "object/old-data";
|
||||
let body_path = format!("{path}/part.1");
|
||||
let receipt_path = format!("{path}/{}", cleanup_runtime::OLD_DATA_CLEANUP_RECEIPT_FILE);
|
||||
wrapper.make_volume(volume).await.unwrap();
|
||||
wrapper
|
||||
.write_all(volume, &body_path, Bytes::from_static(b"old body"))
|
||||
.await
|
||||
.unwrap();
|
||||
let read_lease = wrapper
|
||||
.acquire_snapshot_lease(volume, path)
|
||||
.await
|
||||
.expect("ordinary GET lease");
|
||||
let (entered, entry) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = std::sync::mpsc::channel();
|
||||
let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move {
|
||||
entered.send(()).unwrap();
|
||||
// Deliberately stall the only cleanup scheduler to prove that
|
||||
// ordinary GET completion does not dispatch to it.
|
||||
released
|
||||
.recv_timeout(Duration::from_secs(30))
|
||||
.expect("release stalled scheduler");
|
||||
Ok(())
|
||||
}));
|
||||
tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), wrapper.release_snapshot_lease(volume, path, read_lease))
|
||||
.await
|
||||
.expect("ordinary GET release must progress while the cleanup scheduler is stalled")
|
||||
.expect("release without reclamation");
|
||||
let fence_path = super::super::quota_mutation_fence_path(volume, "object");
|
||||
let fence = wrapper
|
||||
.acquire_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path)
|
||||
.await
|
||||
.unwrap();
|
||||
wrapper
|
||||
.release_snapshot_lease(super::super::RUSTFS_META_BUCKET, &fence_path, fence)
|
||||
.await
|
||||
.unwrap();
|
||||
let timeout = wrapper
|
||||
.track_cleanup_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
std::future::pending::<Result<()>>,
|
||||
Duration::from_millis(10),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(timeout, Err(DiskError::Timeout)));
|
||||
assert!(!wrapper.health.is_faulty(), "cleanup queue delay must not mark a healthy disk faulty");
|
||||
let options = DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
};
|
||||
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
// This is the same entry point used after RPC authentication and
|
||||
// scanner publication-lease validation by handle_delete.
|
||||
let result = wrapper
|
||||
.delete_with_publication_guard(volume, path, options.clone(), Some(Arc::new(DropProbe(drops.clone()))))
|
||||
.await;
|
||||
assert!(matches!(result, Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 1);
|
||||
assert!(matches!(wrapper.delete_data_dir(volume, path, options.clone()).await,
|
||||
Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
|
||||
assert!(matches!(wrapper.write_all(volume, &receipt_path, Bytes::from_static(b"receipt")).await,
|
||||
Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"old body"));
|
||||
release.send(()).unwrap();
|
||||
occupied.await.unwrap().unwrap();
|
||||
wrapper
|
||||
.write_all(volume, &receipt_path, Bytes::from_static(b"receipt"))
|
||||
.await
|
||||
.unwrap();
|
||||
wrapper
|
||||
.delete_with_publication_guard(volume, path, options, None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound)));
|
||||
// Snapshot readers defer old-data cleanup. Saturation must release
|
||||
// their token while retaining reclaim intent, rather than leaking a
|
||||
// live token or performing an unbudgeted delete on the RPC runtime.
|
||||
wrapper
|
||||
.write_all(volume, &body_path, Bytes::from_static(b"leased body"))
|
||||
.await
|
||||
.unwrap();
|
||||
let lease = wrapper.acquire_snapshot_lease(volume, path).await.unwrap();
|
||||
assert_eq!(
|
||||
wrapper
|
||||
.delete_data_dir(
|
||||
volume,
|
||||
path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
DataDirDeleteStatus::Deferred
|
||||
);
|
||||
let (entered, entry) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = tokio::sync::oneshot::channel();
|
||||
let occupied = tokio::spawn(cleanup_runtime::disk_operation(async move {
|
||||
entered.send(()).unwrap();
|
||||
released.await.unwrap();
|
||||
Ok(())
|
||||
}));
|
||||
tokio::time::timeout(Duration::from_secs(20), entry).await.unwrap().unwrap();
|
||||
assert!(matches!(wrapper.release_snapshot_lease(volume, path, lease).await,
|
||||
Err(DiskError::Io(err)) if err.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!(wrapper.read_all(volume, &body_path).await.unwrap(), Bytes::from_static(b"leased body"));
|
||||
release.send(()).unwrap();
|
||||
occupied.await.unwrap().unwrap();
|
||||
assert_eq!(
|
||||
wrapper
|
||||
.delete_data_dir(
|
||||
volume,
|
||||
path,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
DataDirDeleteStatus::Deleted
|
||||
);
|
||||
assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
|
||||
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
|
||||
@@ -223,12 +223,6 @@ pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()
|
||||
std::fs::rename(from, to)
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn read_file(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
|
||||
fs::read(path.as_ref()).await
|
||||
}
|
||||
|
||||
// Bucket existence cache - reduces statx syscalls for repeated bucket checks
|
||||
|
||||
/// Cache for bucket directory existence checks.
|
||||
@@ -614,28 +608,6 @@ mod tests {
|
||||
assert_eq!(content, "test content");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let file_path = temp_dir.path().join("test_read.txt");
|
||||
|
||||
let test_content = b"This is test content for reading";
|
||||
tokio::fs::write(&file_path, test_content).await.unwrap();
|
||||
|
||||
// Read file
|
||||
let read_content = read_file(&file_path).await.unwrap();
|
||||
assert_eq!(read_content, test_content);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file_nonexistent() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let file_path = temp_dir.path().join("nonexistent.txt");
|
||||
|
||||
// Should fail for non-existent file
|
||||
assert!(read_file(&file_path).await.is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_same_file() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
+344
-177
@@ -221,8 +221,10 @@ struct ListingMetadataRead {
|
||||
has_namespace_child_candidate: bool,
|
||||
}
|
||||
|
||||
fn read_all_data_std(path: &Path) -> core::result::Result<(Vec<u8>, Option<OffsetDateTime>), ReadAllError> {
|
||||
let mut file = std::fs::File::open(path).map_err(ReadAllError::Open)?;
|
||||
fn read_all_data_std(
|
||||
opened: std::io::Result<std::fs::File>,
|
||||
) -> core::result::Result<(Vec<u8>, Option<OffsetDateTime>), ReadAllError> {
|
||||
let mut file = opened.map_err(ReadAllError::Open)?;
|
||||
let metadata = file.metadata().map_err(|err| ReadAllError::Disk(to_file_error(err).into()))?;
|
||||
|
||||
if metadata.is_dir() {
|
||||
@@ -5353,7 +5355,7 @@ pub struct LocalDisk {
|
||||
/// pathname is later covered by another mount.
|
||||
io_root: PathBuf,
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease: std::fs::File,
|
||||
mount_lease: Arc<std::fs::File>,
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease_mount_id: Option<u64>,
|
||||
/// Public path for callers that need the configured disk layout. Internal
|
||||
@@ -5610,24 +5612,6 @@ fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option<u64>
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
#[cfg(target_os = "linux")]
|
||||
fn open_mount_lease(root: &Path) -> Result<(std::fs::File, PathBuf, Option<u64>)> {
|
||||
use rustix::fs::{Mode, OFlags, open};
|
||||
use std::os::fd::AsRawFd as _;
|
||||
|
||||
let fd = open(
|
||||
root,
|
||||
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::empty(),
|
||||
)
|
||||
.map_err(std::io::Error::from)
|
||||
.map_err(DiskError::from)?;
|
||||
let lease = std::fs::File::from(fd);
|
||||
let io_root = PathBuf::from(format!("/proc/self/fd/{}/.", lease.as_raw_fd()));
|
||||
let mount_id = mount_id_for_fd(&lease);
|
||||
Ok((lease, io_root, mount_id))
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn open_mount_lease(root: &Path) -> Result<PathBuf> {
|
||||
Ok(root.to_path_buf())
|
||||
@@ -5711,7 +5695,11 @@ impl LocalDisk {
|
||||
let root = publication_root.path().to_path_buf();
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let (mount_lease, io_root, mount_lease_mount_id) = Self::open_mount_lease(&root)?;
|
||||
let (mount_lease, io_root, mount_lease_mount_id) = (
|
||||
publication_root.directory().clone(),
|
||||
publication_root.io_path().to_path_buf(),
|
||||
mount_id_for_fd(publication_root.directory()),
|
||||
);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let io_root = Self::open_mount_lease(&root)?;
|
||||
|
||||
@@ -5759,9 +5747,11 @@ impl LocalDisk {
|
||||
state = "format_path_resolved",
|
||||
"Local disk format path resolved"
|
||||
);
|
||||
let (format_data, format_meta) = read_file_exists(&io_format_path).await.inspect_err(|err| {
|
||||
log_startup_disk_error("read_format_json", &io_format_path, err);
|
||||
})?;
|
||||
let (format_data, format_meta) = read_file_exists(&io_format_path, &publication_root)
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
log_startup_disk_error("read_format_json", &io_format_path, err);
|
||||
})?;
|
||||
|
||||
let mut id = None;
|
||||
// let mut format_legacy = false;
|
||||
@@ -5907,7 +5897,13 @@ impl LocalDisk {
|
||||
|
||||
let io_root = disk.io_root.clone();
|
||||
let publication_root = disk.publication_root.clone();
|
||||
tokio::spawn(Self::cleanup_deleted_objects_loop(io_root, publication_root, exit_rx));
|
||||
tokio::spawn(Self::cleanup_deleted_objects_loop(
|
||||
io_root,
|
||||
publication_root,
|
||||
#[cfg(target_os = "linux")]
|
||||
disk.mount_lease.clone(),
|
||||
exit_rx,
|
||||
));
|
||||
debug!(
|
||||
event = EVENT_DISK_LOCAL_STARTUP_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -5923,30 +5919,53 @@ impl LocalDisk {
|
||||
async fn cleanup_deleted_objects_loop(
|
||||
root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
#[cfg(target_os = "linux")] mount_lease: Arc<std::fs::File>,
|
||||
mut exit_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) {
|
||||
let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL;
|
||||
let mut interval = interval_at(start_at, DELETED_OBJECTS_CLEANUP_INTERVAL);
|
||||
interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = interval.tick() => {
|
||||
if let Err(err) = Self::cleanup_deleted_objects(root.clone()).await {
|
||||
let root = root.clone();
|
||||
let publication_root = publication_root.clone();
|
||||
#[cfg(target_os = "linux")]
|
||||
let mount_lease = mount_lease.clone();
|
||||
if let Err(err) = super::cleanup_runtime::run_gc(move |budget| async move {
|
||||
// Queued/detached scans must keep /proc/self/fd paths
|
||||
// attached to the original mount after LocalDisk drops.
|
||||
#[cfg(target_os = "linux")]
|
||||
let _mount_lease = mount_lease;
|
||||
if let Err(err) = Self::cleanup_deleted_objects(root.clone(), &budget).await {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
task = "deleted_objects",
|
||||
state = "failed",
|
||||
error = ?err,
|
||||
"Disk local background cleanup failed"
|
||||
);
|
||||
}
|
||||
if let Err(err) = Self::cleanup_stale_tmp_objects(root.clone(), &publication_root, &budget).await {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
task = "stale_tmp_objects",
|
||||
state = "failed",
|
||||
error = ?err,
|
||||
"Disk local background cleanup failed"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}).await {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
task = "deleted_objects",
|
||||
state = "failed",
|
||||
error = ?err,
|
||||
"Disk local background cleanup failed"
|
||||
);
|
||||
}
|
||||
if let Err(err) = Self::cleanup_stale_tmp_objects(root.clone(), &publication_root).await {
|
||||
error!(
|
||||
event = EVENT_DISK_LOCAL_BACKGROUND_CLEANUP,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_DISK_LOCAL,
|
||||
task = "stale_tmp_objects",
|
||||
task = "cleanup_dispatch",
|
||||
state = "failed",
|
||||
error = ?err,
|
||||
"Disk local background cleanup failed"
|
||||
@@ -6037,95 +6056,123 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_stale_tmp_objects(root: PathBuf, publication_root: &os::PublicationRoot) -> Result<()> {
|
||||
Self::cleanup_stale_tmp_objects_with_expiry(root, publication_root, STALE_TMP_OBJECT_EXPIRY).await
|
||||
async fn cleanup_stale_tmp_objects(
|
||||
root: PathBuf,
|
||||
publication_root: &os::PublicationRoot,
|
||||
budget: &super::cleanup_runtime::GcBudget,
|
||||
) -> Result<()> {
|
||||
Self::cleanup_stale_tmp_objects_with_expiry(root, publication_root, STALE_TMP_OBJECT_EXPIRY, budget).await
|
||||
}
|
||||
|
||||
async fn cleanup_stale_tmp_objects_with_expiry(
|
||||
root: PathBuf,
|
||||
publication_root: &os::PublicationRoot,
|
||||
expiry: Duration,
|
||||
budget: &super::cleanup_runtime::GcBudget,
|
||||
) -> Result<()> {
|
||||
let tmp_path = Self::meta_path(&root, RUSTFS_META_TMP_BUCKET);
|
||||
let mut entries = match fs::read_dir(&tmp_path).await {
|
||||
let mut entries = match budget.step(async { Ok(fs::read_dir(&tmp_path).await?) }).await {
|
||||
Ok(entries) => entries,
|
||||
Err(e) => {
|
||||
if e.kind() == ErrorKind::NotFound {
|
||||
if matches!(&e, DiskError::Io(err) if err.kind() == ErrorKind::NotFound) {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(e.into());
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.is_empty() || name == "." || name == ".." || name == ".trash" {
|
||||
continue;
|
||||
loop {
|
||||
let more = budget
|
||||
.step(async {
|
||||
let Some(entry) = entries.next_entry().await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
Self::cleanup_stale_tmp_entry(entry, &root, publication_root, expiry).await?;
|
||||
Ok(true)
|
||||
})
|
||||
.await?;
|
||||
if !more {
|
||||
break;
|
||||
}
|
||||
|
||||
let file_type = entry.file_type().await?;
|
||||
if !file_type.is_dir() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Some(age) = entry
|
||||
.metadata()
|
||||
.await?
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|modified| modified.elapsed().ok())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if age <= expiry {
|
||||
continue;
|
||||
}
|
||||
|
||||
let target_path = Self::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET).join(Uuid::new_v4().to_string());
|
||||
rename_all(entry.path(), target_path, Self::meta_path(&root, RUSTFS_META_BUCKET), publication_root).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_deleted_objects(root: PathBuf) -> Result<()> {
|
||||
async fn cleanup_stale_tmp_entry(
|
||||
entry: fs::DirEntry,
|
||||
root: &Path,
|
||||
publication_root: &os::PublicationRoot,
|
||||
expiry: Duration,
|
||||
) -> Result<()> {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.is_empty() || name == "." || name == ".." || name == ".trash" || !entry.file_type().await?.is_dir() {
|
||||
return Ok(());
|
||||
}
|
||||
let Some(age) = entry
|
||||
.metadata()
|
||||
.await?
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|modified| modified.elapsed().ok())
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
if age <= expiry {
|
||||
return Ok(());
|
||||
}
|
||||
let target_path = Self::meta_path(root, RUSTFS_META_TMP_DELETED_BUCKET).join(Uuid::new_v4().to_string());
|
||||
rename_all(entry.path(), target_path, Self::meta_path(root, RUSTFS_META_BUCKET), publication_root).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_deleted_objects(root: PathBuf, budget: &super::cleanup_runtime::GcBudget) -> Result<()> {
|
||||
let trash = Self::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET);
|
||||
let mut entries = match fs::read_dir(&trash).await {
|
||||
let mut entries = match budget.step(async { Ok(fs::read_dir(&trash).await?) }).await {
|
||||
Ok(entries) => entries,
|
||||
Err(e) => {
|
||||
if e.kind() == ErrorKind::NotFound {
|
||||
if matches!(&e, DiskError::Io(err) if err.kind() == ErrorKind::NotFound) {
|
||||
return Ok(());
|
||||
}
|
||||
return Err(e.into());
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.is_empty() || name == "." || name == ".." {
|
||||
continue;
|
||||
}
|
||||
|
||||
let file_type = entry.file_type().await?;
|
||||
|
||||
let path = trash.join(name);
|
||||
|
||||
if file_type.is_dir() {
|
||||
if let Err(e) = tokio::fs::remove_dir_all(path).await
|
||||
&& e.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err(e.into());
|
||||
}
|
||||
} else if let Err(e) = tokio::fs::remove_file(path).await
|
||||
&& e.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err(e.into());
|
||||
loop {
|
||||
let more = budget
|
||||
.step(async {
|
||||
let Some(entry) = entries.next_entry().await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
Self::cleanup_trash_entry(entry, &trash).await?;
|
||||
Ok(true)
|
||||
})
|
||||
.await?;
|
||||
if !more {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_trash_entry(entry: fs::DirEntry, trash: &Path) -> Result<()> {
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if name.is_empty() || name == "." || name == ".." {
|
||||
return Ok(());
|
||||
}
|
||||
let path = trash.join(name);
|
||||
let result = if entry.file_type().await?.is_dir() {
|
||||
fs::remove_dir_all(path).await
|
||||
} else {
|
||||
fs::remove_file(path).await
|
||||
};
|
||||
match result {
|
||||
Err(err) if err.kind() != ErrorKind::NotFound => Err(err.into()),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_valid_volname(volname: &str) -> bool {
|
||||
if volname.len() < 3 {
|
||||
return false;
|
||||
@@ -6252,6 +6299,19 @@ impl LocalDisk {
|
||||
self.local_disk_object_path(self.io_root(), bucket, key)
|
||||
}
|
||||
|
||||
/// Only for operations that open through PublicationRoot or a commit guard:
|
||||
/// containment and symlink checks happen with the actual open, off the runtime.
|
||||
fn io_get_object_open_path(&self, bucket: &str, key: &str) -> Result<PathBuf> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let (bucket_path, path) = build_local_disk_object_path(self.io_root(), bucket, key);
|
||||
check_local_disk_object_path_components(self.io_root(), &bucket_path, &path)?;
|
||||
Ok(path)
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
self.io_get_object_path(bucket, key)
|
||||
}
|
||||
|
||||
fn io_get_bucket_path(&self, bucket: &str) -> Result<PathBuf> {
|
||||
self.local_disk_bucket_path(self.io_root(), bucket)
|
||||
}
|
||||
@@ -6410,7 +6470,7 @@ impl LocalDisk {
|
||||
|
||||
let mut meta = FileMeta::new();
|
||||
if !fi.fresh {
|
||||
let (buf, _) = read_file_exists(&p).await?;
|
||||
let (buf, _) = read_file_exists(&p, &self.publication_root).await?;
|
||||
if !buf.is_empty() {
|
||||
let _ = meta.unmarshal_msg(&buf).map_err(|_| {
|
||||
meta = FileMeta::new();
|
||||
@@ -6436,6 +6496,36 @@ impl LocalDisk {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Keep the common GET completion on its caller. Only a last-reader
|
||||
/// release with pending reclamation needs an owned cleanup task. Leave
|
||||
/// that token untouched so the slow path owns the complete transition.
|
||||
pub(in crate::disk) async fn release_snapshot_lease_without_cleanup(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
token: SnapshotLeaseToken,
|
||||
) -> bool {
|
||||
let key = SnapshotLeaseKey {
|
||||
volume: volume.to_owned(),
|
||||
path: path.to_owned(),
|
||||
};
|
||||
let mut registry = self.snapshot_leases.lock();
|
||||
let Some(entry) = registry.entries.get_mut(&key) else {
|
||||
return true;
|
||||
};
|
||||
if !entry.deleting
|
||||
&& entry.pending_delete.is_some()
|
||||
&& (entry.tokens.is_empty() || (entry.tokens.len() == 1 && entry.tokens.contains(&token)))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
entry.tokens.remove(&token);
|
||||
if entry.tokens.is_empty() && !entry.deleting {
|
||||
registry.entries.remove(&key);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
async fn delete_data_dir_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -7489,10 +7579,11 @@ impl LocalDisk {
|
||||
// - metadata failure -> to_file_error
|
||||
// - parse failure -> propagated verbatim from read_xl_meta_no_data_sync (`?`)
|
||||
let path = file_path.as_ref().to_path_buf();
|
||||
let root = self.publication_root.clone();
|
||||
let (data, modtime) =
|
||||
spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<(Vec<u8>, Option<OffsetDateTime>)> {
|
||||
// Read-only open, equivalent to O_RDONLY (get_readonly_options only sets read(true)).
|
||||
let mut f = std::fs::File::open(&path).map_err(to_file_error)?;
|
||||
let mut f = root.open_readonly(&path).map_err(to_file_error)?;
|
||||
|
||||
let meta = f.metadata().map_err(to_file_error)?;
|
||||
|
||||
@@ -7546,6 +7637,7 @@ impl LocalDisk {
|
||||
let metadata_path = self.io_get_object_path(bucket, path.to_string_lossy().as_ref());
|
||||
// A part's existence check, metadata read and decode share one dispatch.
|
||||
// Keep open errors unmapped for the existing missing-volume fallback.
|
||||
let root = self.publication_root.clone();
|
||||
let result = spawn_blocking_with_disk_io_permit(io_permit, move || -> Result<_> {
|
||||
let part_error = |error: String| ObjectPartInfo {
|
||||
number: num,
|
||||
@@ -7558,7 +7650,7 @@ impl LocalDisk {
|
||||
// Invalid metadata paths remain request errors, but missing data wins
|
||||
// first, as it does in the serial reader.
|
||||
let metadata_path = metadata_path?;
|
||||
Ok(read_all_data_std(&metadata_path)
|
||||
Ok(read_all_data_std(root.open_readonly(&metadata_path))
|
||||
.map(|(data, _)| ObjectPartInfo::unmarshal(&data).unwrap_or_else(|err| part_error(err.to_string()))))
|
||||
})
|
||||
.await;
|
||||
@@ -7578,8 +7670,9 @@ impl LocalDisk {
|
||||
let object_dir = self.io_get_object_path(volume, object_name)?;
|
||||
let metadata_path = object_dir.join(STORAGE_FORMAT_FILE);
|
||||
let volume_dir = self.io_get_bucket_path(volume)?;
|
||||
let root = self.publication_root.clone();
|
||||
let result = spawn_blocking_with_disk_io_permit(io_permit, move || {
|
||||
let (bytes, _) = read_all_data_std(&metadata_path)?;
|
||||
let (bytes, _) = read_all_data_std(root.open_readonly(&metadata_path))?;
|
||||
let file_meta = FileMeta::load(&bytes).ok();
|
||||
let data_dirs: HashSet<String> = file_meta
|
||||
.as_ref()
|
||||
@@ -7677,7 +7770,8 @@ impl LocalDisk {
|
||||
// gating the volume fallback on the open error alone is equivalent to
|
||||
// the original code, where the fallback lived solely in the open match arm.
|
||||
let path = file_path.as_ref().to_path_buf();
|
||||
let res = spawn_blocking_with_disk_io_permit(io_permit, move || read_all_data_std(&path))
|
||||
let root = self.publication_root.clone();
|
||||
let res = spawn_blocking_with_disk_io_permit(io_permit, move || read_all_data_std(root.open_readonly(&path)))
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
|
||||
@@ -8505,16 +8599,16 @@ impl LocalDisk {
|
||||
os::make_dir_all(parent, skip_parent).await?;
|
||||
}
|
||||
|
||||
let root = self.publication_root.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&path);
|
||||
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.write(true)
|
||||
.truncate(true)
|
||||
.open(&path)
|
||||
.map_err(to_file_error)?;
|
||||
let (mut file, parent_handle) = if sync == SyncMode::FileAndDir {
|
||||
root.create_truncate_with_parent(&path).map_err(to_file_error)?
|
||||
} else {
|
||||
(root.create_truncate(&path).map_err(to_file_error)?, None)
|
||||
};
|
||||
std::io::Write::write_all(&mut file, buf.as_ref()).map_err(to_file_error)?;
|
||||
if sync != SyncMode::None {
|
||||
file.sync_data().map_err(to_file_error)?;
|
||||
@@ -8523,7 +8617,7 @@ impl LocalDisk {
|
||||
if sync == SyncMode::FileAndDir
|
||||
&& let Some(parent) = path.parent()
|
||||
{
|
||||
os::fsync_dir_std(parent).map_err(to_file_error)?;
|
||||
os::fsync_directory_handle_std(parent, parent_handle.as_ref()).map_err(to_file_error)?;
|
||||
}
|
||||
}
|
||||
Ok::<_, std::io::Error>(())
|
||||
@@ -8548,7 +8642,18 @@ impl LocalDisk {
|
||||
os::make_dir_all(parent, skip_parent).await?;
|
||||
}
|
||||
|
||||
let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?;
|
||||
let f = if mode == O_CREATE | O_WRONLY | O_TRUNC {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let root = self.publication_root.clone();
|
||||
File::from_std(
|
||||
tokio::task::spawn_blocking(move || root.create_truncate(&path))
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map_err(to_file_error)?,
|
||||
)
|
||||
} else {
|
||||
super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?
|
||||
};
|
||||
|
||||
Ok(f)
|
||||
}
|
||||
@@ -9424,9 +9529,9 @@ fn file_meta_counts_toward_limit(meta: &FileMeta) -> bool {
|
||||
}
|
||||
|
||||
// Filter std::io::ErrorKind::NotFound
|
||||
async fn read_file_exists(path: impl AsRef<Path>) -> Result<(Bytes, Option<Metadata>)> {
|
||||
async fn read_file_exists(path: impl AsRef<Path>, root: &os::PublicationRoot) -> Result<(Bytes, Option<Metadata>)> {
|
||||
let p = path.as_ref();
|
||||
let (data, meta) = match read_file_all(&p).await {
|
||||
let (data, meta) = match read_file_all(&p, root).await {
|
||||
Ok((data, meta)) => (data, Some(meta)),
|
||||
Err(e) => {
|
||||
if e == Error::FileNotFound {
|
||||
@@ -9445,32 +9550,28 @@ async fn read_file_exists(path: impl AsRef<Path>) -> Result<(Bytes, Option<Metad
|
||||
Ok((data, meta))
|
||||
}
|
||||
|
||||
async fn read_file_all(path: impl AsRef<Path>) -> Result<(Bytes, Metadata)> {
|
||||
let p = path.as_ref();
|
||||
let meta = read_file_metadata(&path).await?;
|
||||
|
||||
let data = fs::read(&p)
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
log_startup_disk_io_error("read_file_all", p, err);
|
||||
})
|
||||
.map_err(to_file_error)?;
|
||||
|
||||
Ok((data.into(), meta))
|
||||
}
|
||||
|
||||
async fn read_file_metadata(p: impl AsRef<Path>) -> Result<Metadata> {
|
||||
let path = p.as_ref();
|
||||
let meta = fs::metadata(path)
|
||||
.await
|
||||
.inspect_err(|err| {
|
||||
if err.kind() != ErrorKind::NotFound {
|
||||
log_startup_disk_io_error("read_file_metadata", path, err);
|
||||
}
|
||||
})
|
||||
.map_err(to_file_error)?;
|
||||
|
||||
Ok(meta)
|
||||
async fn read_file_all(path: impl AsRef<Path>, root: &os::PublicationRoot) -> Result<(Bytes, Metadata)> {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let root = root.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = (|| {
|
||||
let mut file = root.open_readonly(&path)?;
|
||||
let metadata = file.metadata()?;
|
||||
let mut data = Vec::new();
|
||||
std::io::Read::read_to_end(&mut file, &mut data)?;
|
||||
Ok::<_, std::io::Error>((Bytes::from(data), metadata))
|
||||
})();
|
||||
result
|
||||
.inspect_err(|err| {
|
||||
if err.kind() != ErrorKind::NotFound {
|
||||
log_startup_disk_io_error("read_file_all", &path, err);
|
||||
}
|
||||
})
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
}
|
||||
|
||||
fn skip_access_checks(p: impl AsRef<str>) -> bool {
|
||||
@@ -9552,13 +9653,19 @@ fn check_local_disk_valid_path(root: &Path, path: impl AsRef<Path>) -> Result<()
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn check_local_disk_valid_object_path_at(root: &Path, root_fd: &std::fs::File, bucket_path: &Path, path: &Path) -> Result<()> {
|
||||
check_local_disk_object_path_components(root, bucket_path, path)?;
|
||||
reject_local_disk_symlink_components_at(root, root_fd, &normalize_path_components(path))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn check_local_disk_object_path_components(root: &Path, bucket_path: &Path, path: &Path) -> Result<()> {
|
||||
let bucket_path = normalize_path_components(bucket_path);
|
||||
let path = normalize_path_components(path);
|
||||
if !bucket_path.starts_with(root) || !path.starts_with(&bucket_path) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
|
||||
reject_local_disk_symlink_components_at(root, root_fd, &path)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
@@ -10407,9 +10514,9 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
let p = self.io_get_object_path(volume, path)?;
|
||||
let p = self.io_get_object_open_path(volume, path)?;
|
||||
|
||||
let (data, _) = read_file_all(&p).await?;
|
||||
let (data, _) = read_file_all(&p, &self.publication_root).await?;
|
||||
|
||||
Ok(data)
|
||||
}
|
||||
@@ -11701,8 +11808,17 @@ impl DiskAPI for LocalDisk {
|
||||
entry.deleting = true;
|
||||
opts
|
||||
};
|
||||
// Release the reader token even under saturation. A rejected reclaim
|
||||
// retains pending_delete (and its on-disk receipt) for a later retry.
|
||||
// Do not wait for execution while a releasing reader may own locks.
|
||||
let claim = SnapshotLeaseDeleteClaim::new(&self.snapshot_leases, key);
|
||||
let result = self.delete_unleased(volume, path, &opts).await;
|
||||
let result = match super::cleanup_runtime::try_disk_execution() {
|
||||
Ok(permit) => {
|
||||
let _permit = permit;
|
||||
self.delete_unleased(volume, path, &opts).await
|
||||
}
|
||||
Err(err) => Err(err.into()),
|
||||
};
|
||||
claim.finish(result.is_ok());
|
||||
result
|
||||
}
|
||||
@@ -11976,7 +12092,7 @@ impl DiskAPI for LocalDisk {
|
||||
};
|
||||
|
||||
// if req.metadata_only {}
|
||||
match read_file_all(&fpath).await {
|
||||
match read_file_all(&fpath, &self.publication_root).await {
|
||||
Ok((data, meta)) => {
|
||||
found += 1;
|
||||
|
||||
@@ -13572,10 +13688,15 @@ mod test {
|
||||
disk.wait_for_startup_cleanup().await;
|
||||
assert_eq!(disk.startup_cleanup_ready.load(Ordering::Acquire), 1);
|
||||
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().join("missing-root"), &publication_root, Duration::ZERO)
|
||||
.await
|
||||
.expect("missing tmp path should be a cleanup no-op");
|
||||
LocalDisk::cleanup_deleted_objects(dir.path().join("missing-root"))
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(
|
||||
dir.path().join("missing-root"),
|
||||
&publication_root,
|
||||
Duration::ZERO,
|
||||
&crate::disk::cleanup_runtime::GcBudget::default(),
|
||||
)
|
||||
.await
|
||||
.expect("missing tmp path should be a cleanup no-op");
|
||||
LocalDisk::cleanup_deleted_objects(dir.path().join("missing-root"), &crate::disk::cleanup_runtime::GcBudget::default())
|
||||
.await
|
||||
.expect("missing trash path should be a cleanup no-op");
|
||||
|
||||
@@ -13588,9 +13709,14 @@ mod test {
|
||||
fs::create_dir_all(&trash_root).await.expect("trash dir should be created");
|
||||
backdate_mtime(&stale_dir, Duration::from_secs(10));
|
||||
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::ZERO)
|
||||
.await
|
||||
.expect("stale tmp directory should move to trash");
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(
|
||||
dir.path().to_path_buf(),
|
||||
&publication_root,
|
||||
Duration::ZERO,
|
||||
&crate::disk::cleanup_runtime::GcBudget::default(),
|
||||
)
|
||||
.await
|
||||
.expect("stale tmp directory should move to trash");
|
||||
assert!(!stale_dir.exists(), "stale tmp directory should be moved away");
|
||||
assert!(live_file.exists(), "plain tmp files should be ignored by stale dir cleanup");
|
||||
|
||||
@@ -13600,7 +13726,7 @@ mod test {
|
||||
fs::create_dir_all(trash_root.join("trash-dir"))
|
||||
.await
|
||||
.expect("trash dir should be created");
|
||||
LocalDisk::cleanup_deleted_objects(dir.path().to_path_buf())
|
||||
LocalDisk::cleanup_deleted_objects(dir.path().to_path_buf(), &crate::disk::cleanup_runtime::GcBudget::default())
|
||||
.await
|
||||
.expect("trash cleanup should remove files and directories");
|
||||
assert!(
|
||||
@@ -20705,9 +20831,14 @@ mod test {
|
||||
// Backdate after the write above: creating stale/data refreshes the
|
||||
// scanned tmp/stale directory's mtime.
|
||||
backdate_mtime(&tmp.join("stale"), Duration::from_secs(10));
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::ZERO)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(
|
||||
dir.path().to_path_buf(),
|
||||
&publication_root,
|
||||
Duration::ZERO,
|
||||
&crate::disk::cleanup_runtime::GcBudget::default(),
|
||||
)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
|
||||
assert!(!tmp.join("stale").exists());
|
||||
assert!(trash.exists());
|
||||
@@ -20734,9 +20865,14 @@ mod test {
|
||||
fs::write(&fresh_dir, b"temporary").await.expect("operation should succeed");
|
||||
fs::write(®ular_file, b"keep").await.expect("operation should succeed");
|
||||
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(dir.path().to_path_buf(), &publication_root, Duration::from_secs(60))
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
LocalDisk::cleanup_stale_tmp_objects_with_expiry(
|
||||
dir.path().to_path_buf(),
|
||||
&publication_root,
|
||||
Duration::from_secs(60),
|
||||
&crate::disk::cleanup_runtime::GcBudget::default(),
|
||||
)
|
||||
.await
|
||||
.expect("operation should succeed");
|
||||
|
||||
assert!(tmp.join("fresh").exists());
|
||||
assert!(regular_file.exists());
|
||||
@@ -20745,6 +20881,44 @@ mod test {
|
||||
assert!(entries.next_entry().await.expect("operation should succeed").is_none());
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn background_cleanup_keeps_mount_lease_after_disk_drop() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().unwrap()).unwrap();
|
||||
let disk = LocalDisk::new(&endpoint, false).await.unwrap();
|
||||
let root = disk.io_root.clone();
|
||||
let trash_entry = LocalDisk::meta_path(&root, RUSTFS_META_TMP_DELETED_BUCKET).join("detached-gc");
|
||||
fs::create_dir_all(&trash_entry).await.unwrap();
|
||||
fs::write(trash_entry.join("part.1"), b"garbage").await.unwrap();
|
||||
let live_file = root.join("keep");
|
||||
fs::write(&live_file, b"live").await.unwrap();
|
||||
let (exit, exited) = tokio::sync::broadcast::channel(1);
|
||||
tokio::time::pause();
|
||||
let cleanup = tokio::spawn(LocalDisk::cleanup_deleted_objects_loop(
|
||||
root.clone(),
|
||||
disk.publication_root.clone(),
|
||||
disk.mount_lease.clone(),
|
||||
exited,
|
||||
));
|
||||
tokio::task::yield_now().await;
|
||||
drop(disk);
|
||||
tokio::task::yield_now().await;
|
||||
assert!(live_file.exists(), "the GC owner must retain the original /proc/self/fd root");
|
||||
tokio::time::advance(DELETED_OBJECTS_CLEANUP_INTERVAL + Duration::from_secs(1)).await;
|
||||
tokio::time::resume();
|
||||
timeout(Duration::from_secs(20), async {
|
||||
while trash_entry.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("detached cleanup must finish against the pinned mount");
|
||||
assert_eq!(fs::read(&live_file).await.unwrap(), b"live");
|
||||
exit.send(()).unwrap();
|
||||
timeout(Duration::from_secs(20), cleanup).await.unwrap().unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cleanup_loop_interval_does_not_tick_immediately() {
|
||||
let start_at = Instant::now() + DELETED_OBJECTS_CLEANUP_INTERVAL;
|
||||
@@ -23887,10 +24061,12 @@ mod test {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file_exists() {
|
||||
let test_file = "./test_read_exists.txt";
|
||||
let directory = tempfile::tempdir().expect("test directory");
|
||||
let root = os::PublicationRoot::new(directory.path()).expect("publication root");
|
||||
let test_file = &directory.path().join("read_exists");
|
||||
|
||||
// Test non-existent file
|
||||
let (data, metadata) = read_file_exists(test_file).await.expect("operation should succeed");
|
||||
let (data, metadata) = read_file_exists(test_file, &root).await.expect("operation should succeed");
|
||||
assert!(data.is_empty());
|
||||
assert!(metadata.is_none());
|
||||
|
||||
@@ -23898,7 +24074,7 @@ mod test {
|
||||
fs::write(test_file, b"test content").await.expect("operation should succeed");
|
||||
|
||||
// Test existing file
|
||||
let (data, metadata) = read_file_exists(test_file).await.expect("operation should succeed");
|
||||
let (data, metadata) = read_file_exists(test_file, &root).await.expect("operation should succeed");
|
||||
assert_eq!(data.as_ref(), b"test content");
|
||||
assert!(metadata.is_some());
|
||||
|
||||
@@ -23908,14 +24084,16 @@ mod test {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file_all() {
|
||||
let test_file = "./test_read_all.txt";
|
||||
let directory = tempfile::tempdir().expect("test directory");
|
||||
let root = os::PublicationRoot::new(directory.path()).expect("publication root");
|
||||
let test_file = &directory.path().join("read_all");
|
||||
let test_content = b"test content for read_all";
|
||||
|
||||
// Create test file
|
||||
fs::write(test_file, test_content).await.expect("operation should succeed");
|
||||
|
||||
// Test reading file
|
||||
let (data, metadata) = read_file_all(test_file).await.expect("operation should succeed");
|
||||
let (data, metadata) = read_file_all(test_file, &root).await.expect("operation should succeed");
|
||||
assert_eq!(data.as_ref(), test_content);
|
||||
assert!(metadata.is_file());
|
||||
assert_eq!(metadata.len(), test_content.len() as u64);
|
||||
@@ -23924,22 +24102,6 @@ mod test {
|
||||
let _ = fs::remove_file(test_file).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_read_file_metadata() {
|
||||
let test_file = "./test_metadata.txt";
|
||||
|
||||
// Create test file
|
||||
fs::write(test_file, b"test").await.expect("operation should succeed");
|
||||
|
||||
// Test reading metadata
|
||||
let metadata = read_file_metadata(test_file).await.expect("operation should succeed");
|
||||
assert!(metadata.is_file());
|
||||
assert_eq!(metadata.len(), 4); // "test" is 4 bytes
|
||||
|
||||
// Clean up
|
||||
let _ = fs::remove_file(test_file).await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_root_path() {
|
||||
// Unix root path
|
||||
@@ -24528,9 +24690,14 @@ mod test {
|
||||
Bytes::from_static(b"later")
|
||||
);
|
||||
|
||||
disk.release_snapshot_lease(volume, &data_dir, renewed)
|
||||
.await
|
||||
.expect("renewed lease release should succeed");
|
||||
assert!(
|
||||
disk.release_snapshot_lease_without_cleanup(volume, &data_dir, renewed).await,
|
||||
"a non-final reader release must not dispatch cleanup"
|
||||
);
|
||||
assert!(
|
||||
!disk.release_snapshot_lease_without_cleanup(volume, &data_dir, second).await,
|
||||
"the final reader must retain its token for owned deferred reclamation"
|
||||
);
|
||||
assert!(
|
||||
disk.read_all(volume, &first_part).await.is_ok(),
|
||||
"one remaining lease must keep the data directory"
|
||||
|
||||
@@ -47,6 +47,33 @@ use tokio::fs;
|
||||
use tracing::{info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
fn transitioned_old_current_source(
|
||||
dst_meta_existed: bool,
|
||||
dst_meta_unparsable: bool,
|
||||
xlmeta: &FileMeta,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
version_id: Uuid,
|
||||
) -> (bool, Option<FileInfo>) {
|
||||
if dst_meta_unparsable {
|
||||
return (false, None);
|
||||
}
|
||||
if !dst_meta_existed {
|
||||
return (true, None);
|
||||
}
|
||||
let Ok(old_current) = xlmeta.into_fileinfo(volume, path, "", false, false, true) else {
|
||||
return (false, None);
|
||||
};
|
||||
if old_current.version_id.unwrap_or_default() != version_id {
|
||||
return (true, None);
|
||||
}
|
||||
if old_current.transition_status == rustfs_filemeta::TRANSITION_COMPLETE {
|
||||
(true, Some(old_current))
|
||||
} else {
|
||||
(true, None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Hold later repair publications after admitting one baseline object. The
|
||||
/// fixture arms this on one replacement disk before rejoining the cluster.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
@@ -214,7 +241,7 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
publication_root: &os::PublicationRoot,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<os::RenameCommitGuard> {
|
||||
#[cfg(windows)]
|
||||
#[cfg(any(windows, target_os = "linux"))]
|
||||
let result = {
|
||||
let source_parent = source_parent.to_path_buf();
|
||||
let destination_parent = destination_parent.to_path_buf();
|
||||
@@ -222,7 +249,7 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
let publication_root = publication_root.clone();
|
||||
os::run_blocking_namespace_operation(mutation_lease, move || {
|
||||
let result = os::prepare_rename_commit_guard(&source_parent, &destination_parent, &base_dir, &publication_root);
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, windows))]
|
||||
if result.is_ok() {
|
||||
run_destination_commit_directory_preparation(&destination_parent);
|
||||
}
|
||||
@@ -230,7 +257,7 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
})
|
||||
.await
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let result = {
|
||||
let _ = mutation_lease;
|
||||
os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root)
|
||||
@@ -244,12 +271,12 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
result.map_err(to_file_error).map_err(DiskError::from)
|
||||
}
|
||||
|
||||
async fn read_rename_destination_metadata(
|
||||
async fn prepare_rename_destination_metadata(
|
||||
file_path: &Path,
|
||||
rename_commit_guard: &os::RenameCommitGuard,
|
||||
mutation_lease: Arc<os::NamespaceMutationLease>,
|
||||
) -> Result<Option<Bytes>> {
|
||||
#[cfg(windows)]
|
||||
#[cfg(any(windows, target_os = "linux"))]
|
||||
let result = {
|
||||
let file_path = file_path.to_path_buf();
|
||||
let rename_commit_guard = rename_commit_guard.clone();
|
||||
@@ -258,10 +285,10 @@ async fn read_rename_destination_metadata(
|
||||
})
|
||||
.await
|
||||
};
|
||||
#[cfg(not(windows))]
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let _ = (rename_commit_guard, mutation_lease);
|
||||
#[cfg(not(windows))]
|
||||
let result = match super::super::fs::read_file(file_path).await {
|
||||
#[cfg(not(any(windows, target_os = "linux")))]
|
||||
let result = match fs::read(file_path).await {
|
||||
Ok(data) => Ok(Some(data)),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => Ok(None),
|
||||
Err(err) => Err(err),
|
||||
@@ -398,7 +425,7 @@ impl LocalDisk {
|
||||
|
||||
// xl.meta path
|
||||
let src_file_path = self.io_get_object_path(src_volume, format!("{}/{}", src_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
let dst_file_path = self.io_get_object_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
let dst_file_path = self.io_get_object_open_path(dst_volume, format!("{}/{}", dst_path, STORAGE_FORMAT_FILE).as_str())?;
|
||||
|
||||
// data_dir path
|
||||
let has_data_dir_path = {
|
||||
@@ -453,8 +480,7 @@ impl LocalDisk {
|
||||
fs::create_dir_all(src_file_parent).await.map_err(to_file_error)?;
|
||||
}
|
||||
// Acquire the common trees before reading destination metadata. On
|
||||
// Windows this pins the object directory identity across metadata
|
||||
// preparation, data publication, rollback backup, and final commit.
|
||||
// Windows this pins the object directory identity across publication.
|
||||
let rename_commit_guard = lock_rename_commit_directories(
|
||||
src_file_parent,
|
||||
dst_file_parent,
|
||||
@@ -472,9 +498,12 @@ impl LocalDisk {
|
||||
source_volume_dir: &src_volume_dir,
|
||||
data_paths: has_data_dir_path.as_ref(),
|
||||
};
|
||||
let has_dst_buf =
|
||||
read_rename_destination_metadata(commit.destination_metadata, &commit.directory_guard, commit.mutation_lease.clone())
|
||||
.await?;
|
||||
let has_dst_buf = prepare_rename_destination_metadata(
|
||||
commit.destination_metadata,
|
||||
&commit.directory_guard,
|
||||
commit.mutation_lease.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
if no_inline {
|
||||
// Non-inline: read xl.meta, parse, write, rename data dir, rename xl.meta
|
||||
@@ -508,6 +537,14 @@ impl LocalDisk {
|
||||
}
|
||||
|
||||
let version_id = fi.version_id.unwrap_or_default();
|
||||
let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
|
||||
has_dst_buf.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
version_id,
|
||||
);
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
|
||||
let rollback_data_dir = has_old_data_dir.or_else(|| {
|
||||
@@ -798,7 +835,9 @@ impl LocalDisk {
|
||||
&& let Some(parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit(parent, Some(commit.mutation_lease.clone())).await {
|
||||
if let Err(err) =
|
||||
os::fsync_commit_directory(parent, &commit.directory_guard, commit.mutation_lease.clone(), None).await
|
||||
{
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
|
||||
fsync_started,
|
||||
@@ -897,6 +936,8 @@ impl LocalDisk {
|
||||
cleanup_data_dir: has_old_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
})
|
||||
} else {
|
||||
// Inline metadata preparation is blocking. The transaction lease is
|
||||
@@ -910,6 +951,8 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let dst_path_for_failpoint = dst_path.to_string();
|
||||
let dst_volume_for_source = dst_volume.to_string();
|
||||
let dst_path_for_source = dst_path.to_string();
|
||||
#[cfg(windows)]
|
||||
let source_parent = src_file_parent.to_path_buf();
|
||||
let rename_commit_guard_for_preparation = commit.directory_guard.clone();
|
||||
@@ -954,6 +997,14 @@ impl LocalDisk {
|
||||
};
|
||||
|
||||
let version_id = fi.version_id.unwrap_or_default();
|
||||
let (old_current_source_checked, old_current_source) = transitioned_old_current_source(
|
||||
has_dst_buf.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_for_source,
|
||||
&dst_path_for_source,
|
||||
version_id,
|
||||
);
|
||||
let old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
let old_version_exists = xlmeta.find_version(Some(version_id)).is_ok();
|
||||
let rollback_data_dir = old_data_dir.or_else(|| {
|
||||
@@ -1003,6 +1054,8 @@ impl LocalDisk {
|
||||
old_data_dir,
|
||||
version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
staged_rollback_path,
|
||||
has_dst_buf.is_none(),
|
||||
prepared_metadata_source,
|
||||
@@ -1022,6 +1075,8 @@ impl LocalDisk {
|
||||
cleanup_data_dir,
|
||||
version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
mut local_rollback_path,
|
||||
destination_was_absent,
|
||||
prepared_metadata_source,
|
||||
@@ -1147,10 +1202,11 @@ impl LocalDisk {
|
||||
&& let Some(dst_parent) = dst_file_path.parent()
|
||||
{
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) = os::fsync_dst_dir_group_commit_or_namespace_file_sync_limit(
|
||||
if let Err(err) = os::fsync_commit_directory(
|
||||
dst_parent,
|
||||
&commit.directory_guard,
|
||||
commit.mutation_lease.clone(),
|
||||
admission,
|
||||
Some(admission),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1262,6 +1318,8 @@ impl LocalDisk {
|
||||
cleanup_data_dir,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1953,10 +1953,7 @@ async fn barrier(volume: &str, object: &str, transaction: Uuid, stage: &str, dri
|
||||
.map_err(to_file_error)?
|
||||
== b"io_error"
|
||||
{
|
||||
return Err(DiskError::Io(std::io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"injected delete recovery barrier failure",
|
||||
)));
|
||||
return Err(DiskError::Io(std::io::Error::other("injected delete recovery barrier failure")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
// #730: disk abstractions still carry staged health and direct-I/O migration paths.
|
||||
|
||||
pub(crate) mod cleanup_runtime;
|
||||
pub mod disk_store;
|
||||
pub mod endpoint;
|
||||
pub mod error;
|
||||
@@ -1776,6 +1777,14 @@ pub struct RenameDataResp {
|
||||
/// "cannot vote", never as `Absent`.
|
||||
#[serde(default)]
|
||||
pub old_current_size: Option<OldCurrentSize>,
|
||||
/// Whether this disk decoded enough of the overwritten current version to
|
||||
/// decide if a transitioned source has to be returned.
|
||||
#[serde(default)]
|
||||
pub old_current_source_checked: bool,
|
||||
/// Transitioned old-current source used by PUT free-version cleanup receipt.
|
||||
/// Empty when the old current is not transitioned or no source was known.
|
||||
#[serde(default)]
|
||||
pub old_current_source: Option<FileInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
@@ -2517,6 +2526,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(uuid),
|
||||
sign: Some(signature.clone()),
|
||||
old_current_size: Some(OldCurrentSize::Present(42)),
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
|
||||
assert_eq!(resp.old_data_dir, Some(uuid));
|
||||
@@ -2537,6 +2548,8 @@ mod tests {
|
||||
cleanup_data_dir: Some(Uuid::new_v4()),
|
||||
sign: Some(vec![0x01, 0x02, 0x03]),
|
||||
old_current_size,
|
||||
old_current_source_checked: false,
|
||||
old_current_source: None,
|
||||
};
|
||||
|
||||
let encoded = rmp_serde::encode::to_vec_named(&resp).expect("named msgpack should encode");
|
||||
@@ -2574,6 +2587,8 @@ mod tests {
|
||||
assert_eq!(decoded.cleanup_data_dir, None);
|
||||
assert_eq!(decoded.sign, legacy.sign);
|
||||
assert_eq!(decoded.old_current_size, None);
|
||||
assert!(!decoded.old_current_source_checked);
|
||||
assert!(decoded.old_current_source.is_none());
|
||||
}
|
||||
|
||||
/// Test constants
|
||||
|
||||
+836
-73
File diff suppressed because it is too large
Load Diff
@@ -1006,6 +1006,9 @@ pub struct ObjectOptions {
|
||||
/// Proves an upper layer holds the bucket lifecycle sentinel. A separate
|
||||
/// fence avoids recursively acquiring the read lock behind a queued writer.
|
||||
pub bucket_lifecycle_lock_fence: Option<NamespaceLockFence>,
|
||||
/// Retains the actual lifecycle guard for an owned multipart commit. A
|
||||
/// lock-loss fence alone does not keep a cancelled caller's lock alive.
|
||||
pub bucket_lifecycle_lock_owner: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
pub replication_request: bool,
|
||||
/// True when the inbound request carried the
|
||||
/// `{x-rustfs-,x-minio-}source-proxy-request` header family with the
|
||||
@@ -1165,6 +1168,7 @@ impl std::fmt::Debug for ObjectOptions {
|
||||
.field("delete_mutation_owner", &self.delete_mutation_owner.is_some())
|
||||
.field("namespace_lock_fence", &self.namespace_lock_fence.is_some())
|
||||
.field("bucket_lifecycle_lock_fence", &self.bucket_lifecycle_lock_fence.is_some())
|
||||
.field("bucket_lifecycle_lock_owner", &self.bucket_lifecycle_lock_owner.is_some())
|
||||
.field("replication_request", &self.replication_request)
|
||||
.field("proxy_request", &self.proxy_request)
|
||||
.field("proxy_header_set", &self.proxy_header_set)
|
||||
|
||||
@@ -18,8 +18,8 @@ use super::{
|
||||
encode_rebalance_stop_propagation_record,
|
||||
};
|
||||
use crate::core::pools::{
|
||||
PoolMeta, PoolRebalanceActivationFence, acquire_pool_activation_fleet_proof, acquire_pool_rebalance_activation_locks,
|
||||
pool_meta_has_active_decommission,
|
||||
PoolMeta, PoolRebalanceActivationFence, PoolRebalanceWorkerFence, acquire_pool_activation_fleet_proof,
|
||||
acquire_pool_rebalance_activation_locks, acquire_pool_rebalance_worker_locks, pool_meta_has_active_decommission,
|
||||
};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::object_api::ObjectOptions;
|
||||
@@ -77,10 +77,33 @@ fn ensure_rebalance_activation_pool_meta_allowed(meta: &PoolMeta) -> Result<()>
|
||||
}
|
||||
|
||||
pub(super) enum RebalanceWorkerActivationFence {
|
||||
Ready(Box<PoolRebalanceActivationFence>),
|
||||
Ready(Box<PoolRebalanceWorkerFence>),
|
||||
NotStartedTerminal,
|
||||
}
|
||||
|
||||
async fn persisted_rebalance_requires_worker_activation<S>(pool: Arc<S>, expected_id: &str) -> Result<bool>
|
||||
where
|
||||
S: EcstoreObjectIO,
|
||||
{
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load_with_opts(
|
||||
pool,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if persisted.id != expected_id {
|
||||
return Err(Error::other(format!(
|
||||
"rebalance metadata changed before worker activation: expected {expected_id}, found {}",
|
||||
persisted.id
|
||||
)));
|
||||
}
|
||||
Ok(crate::services::rebalance::rebalance_requires_worker_activation(&persisted))
|
||||
}
|
||||
|
||||
pub(super) struct RebalanceRunGuard {
|
||||
_local_guard: tokio::sync::OwnedRwLockReadGuard<()>,
|
||||
persisted_guard: rustfs_lock::NamespaceLockGuard,
|
||||
@@ -577,48 +600,58 @@ impl ECStore {
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
|
||||
// Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin. These
|
||||
// shared fences admit a peer without waiting for the run's own entries.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
|
||||
// Classify the durable rebalance record while holding both namespace
|
||||
// fences. A terminal record is a no-op and must not depend on the
|
||||
// notification subsystem having published a fleet proof yet.
|
||||
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
|
||||
let mut worker_fence = acquire_pool_rebalance_worker_locks(pool.clone()).await?;
|
||||
let pool_meta = self
|
||||
.load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation")
|
||||
.load_runtime_pool_meta_under_worker_fence(&mut pool_meta_guard, &worker_fence, "rebalance worker activation")
|
||||
.await?;
|
||||
ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
|
||||
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load_with_opts(
|
||||
pool,
|
||||
ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if persisted.id != expected_id {
|
||||
return Err(Error::other(format!(
|
||||
"rebalance metadata changed before worker activation: expected {expected_id}, found {}",
|
||||
persisted.id
|
||||
)));
|
||||
}
|
||||
|
||||
activation_fence.ensure_held()?;
|
||||
if !crate::services::rebalance::rebalance_requires_worker_activation(&persisted) {
|
||||
let needs_activation = persisted_rebalance_requires_worker_activation(pool, expected_id).await?;
|
||||
worker_fence.ensure_held()?;
|
||||
if !needs_activation {
|
||||
return Ok(RebalanceWorkerActivationFence::NotStartedTerminal);
|
||||
}
|
||||
|
||||
// Active worker admission still requires the fail-closed fleet proof.
|
||||
// Attach it immediately before the final fence validation so expiry or
|
||||
// topology changes are checked again at every later commit boundary.
|
||||
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
|
||||
worker_fence.set_fleet_proof(fleet_proof);
|
||||
worker_fence.ensure_held()?;
|
||||
Ok(RebalanceWorkerActivationFence::Ready(Box::new(worker_fence)))
|
||||
}
|
||||
|
||||
pub(super) async fn fence_rebalance_worker_metadata_update<S>(
|
||||
&self,
|
||||
pool: Arc<S>,
|
||||
expected_id: &str,
|
||||
) -> Result<Option<Box<PoolRebalanceActivationFence>>>
|
||||
where
|
||||
S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
|
||||
{
|
||||
// The admission read fences must already be dropped. Reacquire in the
|
||||
// original order and revalidate every durable boundary across that gap.
|
||||
let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
|
||||
pool_meta_guard.ensure_write_safe("rebalance worker metadata update")?;
|
||||
let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
|
||||
let pool_meta = self
|
||||
.load_runtime_pool_meta_under_activation_fence(
|
||||
&mut pool_meta_guard,
|
||||
&activation_fence,
|
||||
"rebalance worker metadata update",
|
||||
)
|
||||
.await?;
|
||||
ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
|
||||
let needs_activation = persisted_rebalance_requires_worker_activation(pool, expected_id).await?;
|
||||
activation_fence.ensure_held()?;
|
||||
if !needs_activation {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
|
||||
activation_fence.set_fleet_proof(fleet_proof);
|
||||
activation_fence.ensure_held()?;
|
||||
|
||||
Ok(RebalanceWorkerActivationFence::Ready(Box::new(activation_fence)))
|
||||
Ok(Some(Box::new(activation_fence)))
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip_all)]
|
||||
@@ -1816,6 +1849,107 @@ mod tests {
|
||||
assert_activation_locks_released(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_init_missing_fleet_proof_preserves_metadata_and_rechecks_competing_run() {
|
||||
let previous_id = "completed-before-capability-refresh";
|
||||
let terminal = RebalanceMeta {
|
||||
id: previous_id.to_string(),
|
||||
stopped_at: Some(OffsetDateTime::now_utc()),
|
||||
pool_stats: vec![RebalanceStats::default(); 2],
|
||||
..Default::default()
|
||||
};
|
||||
let (_temp_dirs, store, competing_store) = crate::services::rebalance::test_two_pool_stores(Some(terminal.clone())).await;
|
||||
let disk_stats = vec![
|
||||
DiskStat {
|
||||
total_space: 100,
|
||||
available_space: 0,
|
||||
},
|
||||
DiskStat {
|
||||
total_space: 100,
|
||||
available_space: 100,
|
||||
},
|
||||
];
|
||||
let initial_epoch = store.ctx.data_movement_operation_epoch();
|
||||
let proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
|
||||
set_rebalance_disk_stats_override_for_test(store.id, disk_stats.clone());
|
||||
let err = store
|
||||
.init_rebalance_start(vec!["waiting-request".to_string()])
|
||||
.await
|
||||
.expect_err("initialization must fail closed while the live proof is missing");
|
||||
assert!(
|
||||
err.to_string()
|
||||
.contains("pool activation requires a live fleet capability proof")
|
||||
);
|
||||
assert!(
|
||||
crate::core::pools::is_pool_activation_fleet_proof_error(&err),
|
||||
"the admin handler retries init only on this error shape: {err}"
|
||||
);
|
||||
assert_eq!(store.ctx.data_movement_operation_epoch(), initial_epoch);
|
||||
{
|
||||
let local = store.rebalance_meta.read().await;
|
||||
let local = local.as_ref().expect("failed initialization must preserve the previous run");
|
||||
assert_eq!(local.id, previous_id);
|
||||
assert_eq!(local.stopped_at, terminal.stopped_at);
|
||||
assert!(local.cancel.is_none(), "failed initialization must not start a worker");
|
||||
}
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load(store.pools[0].clone())
|
||||
.await
|
||||
.expect("failed initialization must leave durable metadata readable");
|
||||
assert_eq!(persisted.id, previous_id);
|
||||
assert_eq!(persisted.stopped_at, terminal.stopped_at);
|
||||
|
||||
drop(proof_guard);
|
||||
set_rebalance_disk_stats_override_for_test(competing_store.id, disk_stats.clone());
|
||||
let winning_id = competing_store
|
||||
.init_rebalance_start(vec!["winning-request".to_string()])
|
||||
.await
|
||||
.expect("a ready request must be able to initialize one run after the failed attempt");
|
||||
assert_ne!(winning_id, previous_id);
|
||||
|
||||
// The waiting request still holds its old local snapshot. Its retry must
|
||||
// recheck durable state under the activation locks before publishing.
|
||||
set_rebalance_disk_stats_override_for_test(store.id, disk_stats);
|
||||
let remote_conflict = store
|
||||
.init_rebalance_start(vec!["waiting-request".to_string()])
|
||||
.await
|
||||
.expect_err("the waiting request must not replace a durable competing run");
|
||||
assert!(
|
||||
matches!(
|
||||
crate::data_movement::data_movement_stage_source(&remote_conflict),
|
||||
Some(Error::RebalanceAlreadyRunning)
|
||||
),
|
||||
"the save-stage error must retain the typed competing-run conflict: {remote_conflict:?}"
|
||||
);
|
||||
let local_conflict = competing_store
|
||||
.init_rebalance_start(vec!["another-request".to_string()])
|
||||
.await
|
||||
.expect_err("a local retry must not replace its active run");
|
||||
assert!(
|
||||
matches!(local_conflict, Error::RebalanceAlreadyRunning),
|
||||
"the local admission check must report the competing-run conflict: {local_conflict:?}"
|
||||
);
|
||||
persisted
|
||||
.load(store.pools[0].clone())
|
||||
.await
|
||||
.expect("the competing run must remain durable after both rejected retries");
|
||||
assert_eq!(persisted.id, winning_id);
|
||||
assert_eq!(persisted.pool_stats[0].buckets, vec!["winning-request".to_string()]);
|
||||
assert_eq!(
|
||||
store
|
||||
.rebalance_meta
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.expect("old local run must survive")
|
||||
.id,
|
||||
previous_id
|
||||
);
|
||||
assert_eq!(store.ctx.data_movement_operation_epoch(), initial_epoch + 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_skips_terminal_metadata_without_fleet_proof() {
|
||||
@@ -1874,6 +2008,128 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_worker_write_upgrade_revalidates_durable_state(change: &str) {
|
||||
let rebalance_id = "rebalance-worker-write-upgrade";
|
||||
let active = RebalanceMeta {
|
||||
id: rebalance_id.to_string(),
|
||||
percent_free_goal: 0.5,
|
||||
pool_stats: vec![
|
||||
RebalanceStats {
|
||||
participating: true,
|
||||
init_capacity: 100,
|
||||
info: RebalanceInfo {
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
RebalanceStats::default(),
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
let (_temp_dirs, store, peer) =
|
||||
crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(active.clone())).await;
|
||||
let arrived = Arc::new(tokio::sync::Notify::new());
|
||||
let release = Arc::new(tokio::sync::Notify::new());
|
||||
let worker = Arc::clone(&store);
|
||||
let mut tasks = tokio::task::JoinSet::new();
|
||||
tasks.spawn(
|
||||
super::super::runtime::REBALANCE_WORKER_WRITE_UPGRADE_BARRIER
|
||||
.scope((Arc::clone(&arrived), Arc::clone(&release)), async move {
|
||||
worker.start_rebalance_for_id(rebalance_id).await
|
||||
}),
|
||||
);
|
||||
tokio::time::timeout(std::time::Duration::from_secs(30), arrived.notified())
|
||||
.await
|
||||
.expect("empty-queue activation must release its read fences before requesting write authority");
|
||||
|
||||
let mut changed = active.clone();
|
||||
match change {
|
||||
"stopped" => {
|
||||
changed.stopped_at = Some(OffsetDateTime::now_utc());
|
||||
changed.pool_stats[0].info.status = RebalStatus::Stopped;
|
||||
}
|
||||
"replaced" => changed.id = "replacement-rebalance-run".to_string(),
|
||||
"decommission" => {
|
||||
let mut pool_meta = peer.pool_meta.read().await.clone();
|
||||
pool_meta.pools[0].decommission = Some(crate::core::pools::PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
});
|
||||
pool_meta
|
||||
.save(peer.pools.clone())
|
||||
.await
|
||||
.expect("a competing pool metadata writer must acquire the released pool fence");
|
||||
}
|
||||
_ => panic!("unknown durable state transition"),
|
||||
}
|
||||
changed
|
||||
.save_with_opts(
|
||||
peer.pools[0].clone(),
|
||||
ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("a competing run writer must acquire the released rebalance fence");
|
||||
release.notify_one();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(30), tasks.join_next())
|
||||
.await
|
||||
.expect("activation must finish after the durable transition")
|
||||
.expect("activation task must be tracked")
|
||||
.expect("activation must not panic");
|
||||
match change {
|
||||
"stopped" => result.expect("a durably stopped run must become an activation no-op"),
|
||||
"replaced" => assert!(
|
||||
result
|
||||
.expect_err("a replaced run must reject stale activation")
|
||||
.to_string()
|
||||
.contains("expected"),
|
||||
),
|
||||
"decommission" => assert!(matches!(result, Err(Error::DecommissionAlreadyRunning))),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
let mut persisted = RebalanceMeta::new();
|
||||
persisted
|
||||
.load(peer.pools[0].clone())
|
||||
.await
|
||||
.expect("the competing durable state must remain readable");
|
||||
assert_eq!(persisted.id, changed.id, "activation must not overwrite the run identity");
|
||||
assert_eq!(persisted.stopped_at, changed.stopped_at, "activation must not overwrite the stop marker");
|
||||
assert_eq!(persisted.pool_stats[0].info.status, changed.pool_stats[0].info.status);
|
||||
assert!(
|
||||
store
|
||||
.rebalance_meta
|
||||
.read()
|
||||
.await
|
||||
.as_ref()
|
||||
.expect("local metadata must remain present")
|
||||
.cancel
|
||||
.is_none(),
|
||||
"rejected activation must not install a worker token",
|
||||
);
|
||||
assert_activation_locks_released(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_write_upgrade_rechecks_stop() {
|
||||
assert_worker_write_upgrade_revalidates_durable_state("stopped").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_write_upgrade_rechecks_run_identity() {
|
||||
assert_worker_write_upgrade_revalidates_durable_state("replaced").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_worker_write_upgrade_rechecks_decommission() {
|
||||
assert_worker_write_upgrade_revalidates_durable_state("decommission").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() {
|
||||
|
||||
@@ -1440,7 +1440,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn real_rebalance_entry_progresses_while_peer_activation_waits_for_run_fence() {
|
||||
async fn real_rebalance_peer_activates_while_entry_retains_run_fence() {
|
||||
const REBALANCE_ID: &str = "rebalance-peer-activation-lock-order";
|
||||
let (_temp_dirs, store, peer) = crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(Some(
|
||||
active_rebalance_meta(REBALANCE_ID),
|
||||
@@ -1456,7 +1456,7 @@ mod tests {
|
||||
let bucket = crate::disk::RUSTFS_META_BUCKET;
|
||||
let object = "rebalance-peer-activation-object";
|
||||
let version_id = uuid::Uuid::new_v4();
|
||||
let payload = b"entry must drain before peer activation takes the pool fence".repeat(1024);
|
||||
let payload = b"peer activation must not wait for this entry to drain".repeat(1024);
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let target_set = store.pools[1].get_disks_by_key(object);
|
||||
let opts = ObjectOptions {
|
||||
@@ -1516,7 +1516,12 @@ mod tests {
|
||||
);
|
||||
tokio::time::timeout(StdDuration::from_secs(30), attempted.notified())
|
||||
.await
|
||||
.expect("peer activation must attempt the persisted rebalance write fence");
|
||||
.expect("peer activation must attempt the persisted rebalance fence");
|
||||
tokio::time::timeout(StdDuration::from_secs(30), activation_result)
|
||||
.await
|
||||
.expect("peer activation must finish while the entry remains paused")
|
||||
.expect("peer activation result should be sent")
|
||||
.expect("joining the same run must not require its active entry to release the run fence");
|
||||
release.notify_one();
|
||||
|
||||
tokio::time::timeout(StdDuration::from_secs(30), async {
|
||||
@@ -1531,10 +1536,6 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect("entry and peer activation must both make progress");
|
||||
activation_result
|
||||
.await
|
||||
.expect("peer activation result should be sent")
|
||||
.expect("peer activation must not time out behind the entry it blocks");
|
||||
|
||||
let mut reader = target_set
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
|
||||
@@ -27,6 +27,11 @@ use tokio::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
#[cfg(test)]
|
||||
tokio::task_local! {
|
||||
pub(super) static REBALANCE_WORKER_WRITE_UPGRADE_BARRIER: (Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>);
|
||||
}
|
||||
|
||||
pub(super) fn should_fail_repeated_rebalance_bucket_defer(
|
||||
deferred_buckets: &mut HashSet<String>,
|
||||
bucket: &str,
|
||||
@@ -213,6 +218,22 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if must_persist {
|
||||
// Shared admission grants no write authority. Release it before
|
||||
// taking the exclusive metadata fences, then recheck stop/run/pool
|
||||
// state before publishing the locally staged terminal candidate.
|
||||
drop(activation_fence);
|
||||
#[cfg(test)]
|
||||
if let Ok((arrived, release)) = REBALANCE_WORKER_WRITE_UPGRADE_BARRIER.try_with(Clone::clone) {
|
||||
arrived.notify_one();
|
||||
release.notified().await;
|
||||
}
|
||||
let Some(activation_fence) = self
|
||||
.fence_rebalance_worker_metadata_update(pool.clone(), expected_id.as_ref())
|
||||
.await?
|
||||
else {
|
||||
cancel_tx.cancel();
|
||||
return Ok(false);
|
||||
};
|
||||
let save_result = resolve_rebalance_meta_save_result(
|
||||
self.save_rebalance_meta_under_activation_fence(
|
||||
pool,
|
||||
@@ -242,13 +263,14 @@ impl ECStore {
|
||||
cancel_tx.cancel();
|
||||
return Err(err);
|
||||
}
|
||||
} else {
|
||||
if let Err(err) = activation_fence.ensure_held() {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
|
||||
return Err(err);
|
||||
}
|
||||
drop(activation_fence);
|
||||
}
|
||||
if !must_persist && let Err(err) = activation_fence.ensure_held() {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
|
||||
return Err(err);
|
||||
}
|
||||
drop(activation_fence);
|
||||
|
||||
if activation_outcome != RebalanceLocalActivationOutcome::Started {
|
||||
return Ok(must_persist);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -576,6 +576,7 @@ impl MetadataFanoutDiagnostics {
|
||||
|
||||
pub(in crate::set_disk) fn record_quorum_candidate_latency(&self, path: &'static str, read_quorum: usize) {
|
||||
if let Some(latency) = self.quorum_candidate_latency(read_quorum) {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(path, "reached");
|
||||
rustfs_io_metrics::record_get_object_quorum_reached_latency(path, latency.as_secs_f64());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -166,7 +166,7 @@ use sha2::Sha256;
|
||||
use std::hash::{BuildHasher, Hash, Hasher};
|
||||
use std::mem::{self};
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
@@ -410,6 +410,12 @@ impl Drop for ObjectLockDiagGuard {
|
||||
}
|
||||
|
||||
let hold = self.acquired_at.elapsed();
|
||||
if self.op == "put_object_commit" && self.mode == "write" && rustfs_io_metrics::put_stage_metrics_enabled() {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_COMMIT_NAMESPACE_LOCK_HELD,
|
||||
hold.as_secs_f64() * 1000.0,
|
||||
);
|
||||
}
|
||||
record_object_lock_diag_hold_duration(self.op, self.mode, hold);
|
||||
let threshold = get_object_lock_diag_slow_hold_threshold();
|
||||
if hold >= threshold {
|
||||
@@ -925,16 +931,21 @@ pub(in crate::set_disk) struct GetObjectFileInfo {
|
||||
owned: Option<OwnedGetObjectFileInfo>,
|
||||
shared: Option<Arc<GetObjectMetadataCacheEntry>>,
|
||||
condition_stopped: bool,
|
||||
late_metadata_fanout: Option<LateMetadataFanout>,
|
||||
}
|
||||
|
||||
struct OwnedGetObjectFileInfo {
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
late_metadata_fanout_disks: Option<Vec<Option<DiskStore>>>,
|
||||
}
|
||||
|
||||
type OwnedGetObjectFileInfoParts = (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>, Option<Vec<Option<DiskStore>>>);
|
||||
enum LateMetadataFanout {
|
||||
CapturedDisks(Vec<Option<DiskStore>>),
|
||||
RefreshCurrentSet,
|
||||
}
|
||||
|
||||
type OwnedGetObjectFileInfoParts = (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>, Option<LateMetadataFanout>);
|
||||
|
||||
impl GetObjectFileInfo {
|
||||
fn owned(fi: FileInfo, parts_metadata: Vec<FileInfo>, online_disks: Vec<Option<DiskStore>>) -> Self {
|
||||
@@ -943,10 +954,10 @@ impl GetObjectFileInfo {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
late_metadata_fanout_disks: None,
|
||||
}),
|
||||
shared: None,
|
||||
condition_stopped: false,
|
||||
late_metadata_fanout: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -956,16 +967,9 @@ impl GetObjectFileInfo {
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
late_metadata_fanout_disks: Vec<Option<DiskStore>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
owned: Some(OwnedGetObjectFileInfo {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
late_metadata_fanout_disks: Some(late_metadata_fanout_disks),
|
||||
}),
|
||||
shared: None,
|
||||
condition_stopped: false,
|
||||
}
|
||||
let mut snapshot = Self::owned(fi, parts_metadata, online_disks);
|
||||
snapshot.late_metadata_fanout = Some(LateMetadataFanout::CapturedDisks(late_metadata_fanout_disks));
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn shared(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
|
||||
@@ -974,9 +978,16 @@ impl GetObjectFileInfo {
|
||||
owned: None,
|
||||
shared: Some(entry),
|
||||
condition_stopped,
|
||||
late_metadata_fanout: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn shared_with_late_metadata_refresh(entry: Arc<GetObjectMetadataCacheEntry>) -> Self {
|
||||
let mut snapshot = Self::shared(entry);
|
||||
snapshot.late_metadata_fanout = Some(LateMetadataFanout::RefreshCurrentSet);
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn fi(&self) -> &FileInfo {
|
||||
match (&self.owned, &self.shared) {
|
||||
(Some(snapshot), None) => &snapshot.fi,
|
||||
@@ -1002,9 +1013,7 @@ impl GetObjectFileInfo {
|
||||
}
|
||||
|
||||
fn has_late_metadata_fanout(&self) -> bool {
|
||||
self.owned
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| snapshot.late_metadata_fanout_disks.is_some())
|
||||
self.late_metadata_fanout.is_some()
|
||||
}
|
||||
|
||||
fn into_owned(self) -> (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>) {
|
||||
@@ -1013,16 +1022,22 @@ impl GetObjectFileInfo {
|
||||
}
|
||||
|
||||
fn into_owned_with_late_metadata_fanout(self) -> OwnedGetObjectFileInfoParts {
|
||||
match (self.owned, self.shared) {
|
||||
(Some(snapshot), None) => (
|
||||
snapshot.fi,
|
||||
snapshot.parts_metadata,
|
||||
snapshot.online_disks,
|
||||
snapshot.late_metadata_fanout_disks,
|
||||
),
|
||||
let Self {
|
||||
owned,
|
||||
shared,
|
||||
late_metadata_fanout,
|
||||
..
|
||||
} = self;
|
||||
match (owned, shared) {
|
||||
(Some(snapshot), None) => (snapshot.fi, snapshot.parts_metadata, snapshot.online_disks, late_metadata_fanout),
|
||||
(None, Some(entry)) => match Arc::try_unwrap(entry) {
|
||||
Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, None),
|
||||
Err(entry) => (entry.fi.clone(), entry.parts_metadata.clone(), entry.online_disks.clone(), None),
|
||||
Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, late_metadata_fanout),
|
||||
Err(entry) => (
|
||||
entry.fi.clone(),
|
||||
entry.parts_metadata.clone(),
|
||||
entry.online_disks.clone(),
|
||||
late_metadata_fanout,
|
||||
),
|
||||
},
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
@@ -1413,11 +1428,11 @@ mod prepared_get_object_metadata_tests {
|
||||
let snapshot = GetObjectFileInfo::shared(Arc::clone(&cached));
|
||||
|
||||
assert!(snapshot.has_valid_representation());
|
||||
assert!(std::mem::size_of::<GetObjectFileInfo>() >= std::mem::size_of::<OwnedGetObjectFileInfo>());
|
||||
assert!(
|
||||
std::mem::size_of::<GetObjectFileInfo>()
|
||||
<= std::mem::size_of::<OwnedGetObjectFileInfo>() + 2 * std::mem::size_of::<usize>()
|
||||
);
|
||||
// Keep the original payload budget after moving the captured disks out
|
||||
// of OwnedGetObjectFileInfo.
|
||||
let owned_payload_size = std::mem::size_of::<(OwnedGetObjectFileInfo, Option<Vec<Option<DiskStore>>>)>();
|
||||
assert!(std::mem::size_of::<GetObjectFileInfo>() >= owned_payload_size);
|
||||
assert!(std::mem::size_of::<GetObjectFileInfo>() <= owned_payload_size + 2 * std::mem::size_of::<usize>());
|
||||
assert_eq!(Arc::strong_count(&cached), 2, "a cache hit must add one snapshot reference");
|
||||
assert_eq!(snapshot.fi().name, "object");
|
||||
assert_eq!(snapshot.parts_metadata().len(), 1);
|
||||
@@ -4337,6 +4352,9 @@ pub struct SetDisks {
|
||||
/// writes skip the global registry mutex (backlog#1315). `Arc` so clones of
|
||||
/// a set share one generation marker.
|
||||
capacity_dirty_generation: Arc<AtomicU64>,
|
||||
/// Per-set proof that successful rename_data peers return the old-current
|
||||
/// source capability needed before PUT may skip its pre-rename lookup.
|
||||
tier_free_version_rename_data_source_capable: Arc<AtomicBool>,
|
||||
/// Orphan prefixes whose last purge scan met data that can never be
|
||||
/// purged by listing (residue without a committed marker, an in-flight
|
||||
/// write), keyed by `bucket/prefix` with the time of that scan. Empty
|
||||
@@ -4592,6 +4610,14 @@ struct GetObjectMetadataCacheKey {
|
||||
hash: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct GetObjectMetadataCacheLookupKey<'a> {
|
||||
bucket: &'a str,
|
||||
object: &'a str,
|
||||
generation: u64,
|
||||
hash: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct GetObjectMetadataCacheGeneration {
|
||||
index: usize,
|
||||
@@ -4680,6 +4706,33 @@ impl GetObjectMetadataCacheKey {
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> GetObjectMetadataCacheLookupKey<'a> {
|
||||
fn new(bucket: &'a str, object: &'a str, generation: GetObjectMetadataCacheGeneration) -> Self {
|
||||
Self {
|
||||
bucket,
|
||||
object,
|
||||
generation: generation.value,
|
||||
hash: generation.hash,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Hash for GetObjectMetadataCacheLookupKey<'_> {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
self.hash.hash(state);
|
||||
self.generation.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
impl moka::Equivalent<GetObjectMetadataCacheKey> for GetObjectMetadataCacheLookupKey<'_> {
|
||||
fn equivalent(&self, key: &GetObjectMetadataCacheKey) -> bool {
|
||||
self.bucket == key.bucket.as_ref()
|
||||
&& self.object == key.object.as_ref()
|
||||
&& self.generation == key.generation
|
||||
&& self.hash == key.hash
|
||||
}
|
||||
}
|
||||
|
||||
impl Hash for GetObjectMetadataCacheKey {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
self.hash.hash(state);
|
||||
@@ -4690,7 +4743,9 @@ impl Hash for GetObjectMetadataCacheKey {
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum GetObjectMetadataCacheCapability {
|
||||
Full,
|
||||
// Quorum-validated metadata without a complete body or write-target layout.
|
||||
// A quorum-selected layout can serve body reads, but is not a mutation layout.
|
||||
QuorumSelected,
|
||||
// Quorum-validated metadata used only to evaluate a terminal read condition.
|
||||
ConditionOnly,
|
||||
}
|
||||
|
||||
@@ -5233,6 +5288,7 @@ impl SetDisks {
|
||||
ctx,
|
||||
capacity_scope_cache: Arc::new(std::sync::RwLock::new(CapacityScopeCache::default())),
|
||||
capacity_dirty_generation: Arc::new(AtomicU64::new(u64::MAX)),
|
||||
tier_free_version_rename_data_source_capable: Arc::new(AtomicBool::new(false)),
|
||||
orphan_purge_backoff: Arc::new(std::sync::Mutex::new(HashMap::new())),
|
||||
#[cfg(test)]
|
||||
storage_class_config_override: Arc::new(std::sync::RwLock::new(None)),
|
||||
@@ -5247,6 +5303,15 @@ impl SetDisks {
|
||||
&self.ctx
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn tier_free_version_rename_data_source_capable(&self) -> bool {
|
||||
self.tier_free_version_rename_data_source_capable.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_tier_free_version_rename_data_source_capability(&self, capable: bool) {
|
||||
self.tier_free_version_rename_data_source_capable
|
||||
.store(capable, Ordering::Release);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn set_instance_ctx_for_test(&mut self, ctx: Arc<InstanceContext>) {
|
||||
self.local_lock_manager = ctx.lock_manager();
|
||||
|
||||
@@ -31,6 +31,7 @@ use crate::disk::DiskAPI;
|
||||
pub(crate) enum BucketInfoQuorum {
|
||||
Read,
|
||||
Write,
|
||||
HealCandidates,
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
@@ -104,7 +105,9 @@ impl SetDisks {
|
||||
// class. A namespace read must intersect that majority; object
|
||||
// readers still enforce the persisted layout's data-shard quorum.
|
||||
BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count.div_ceil(2).max(1)),
|
||||
BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1),
|
||||
BucketInfoQuorum::Write | BucketInfoQuorum::HealCandidates => {
|
||||
reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, disk_count / 2 + 1)
|
||||
}
|
||||
};
|
||||
if let Some(err) = error {
|
||||
return Err(err.into());
|
||||
@@ -126,6 +129,29 @@ impl SetDisks {
|
||||
self.list_bucket_with_quorum(BucketInfoQuorum::Write).await
|
||||
}
|
||||
|
||||
pub(crate) async fn bucket_needs_reconstruction(&self, bucket: &str) -> Result<bool> {
|
||||
let results = join_all(self.disk_inventory().await.into_iter().map(|disk| async move {
|
||||
match disk {
|
||||
Some(disk) => disk.stat_volume(bucket).await,
|
||||
None => Err(DiskError::DiskNotFound),
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
let mut present = false;
|
||||
let mut missing = false;
|
||||
for result in results {
|
||||
match result {
|
||||
Ok(_) => present = true,
|
||||
Err(DiskError::VolumeNotFound) => missing = true,
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
if !present {
|
||||
return Err(Error::VolumeNotFound);
|
||||
}
|
||||
Ok(missing)
|
||||
}
|
||||
|
||||
pub(crate) async fn list_bucket_with_quorum(&self, quorum: BucketInfoQuorum) -> Result<(Vec<BucketInfo>, bool)> {
|
||||
let disks = self.disk_inventory().await;
|
||||
let write_quorum = (disks.len() / 2) + 1;
|
||||
@@ -134,6 +160,7 @@ impl SetDisks {
|
||||
let required = match quorum {
|
||||
BucketInfoQuorum::Read => disks.len().div_ceil(2).max(1),
|
||||
BucketInfoQuorum::Write => write_quorum,
|
||||
BucketInfoQuorum::HealCandidates => disks.len().max(1),
|
||||
};
|
||||
|
||||
let mut futures = Vec::with_capacity(disks.len());
|
||||
@@ -165,7 +192,9 @@ impl SetDisks {
|
||||
|
||||
let error = match quorum {
|
||||
BucketInfoQuorum::Read => reduce_read_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required),
|
||||
BucketInfoQuorum::Write => reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required),
|
||||
BucketInfoQuorum::Write | BucketInfoQuorum::HealCandidates => {
|
||||
reduce_write_quorum_errs(&errs, BUCKET_OP_IGNORED_ERRS, required)
|
||||
}
|
||||
};
|
||||
if let Some(err) = error {
|
||||
return Err(err.into());
|
||||
@@ -190,10 +219,17 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
|
||||
let minimum_presence = if matches!(quorum, BucketInfoQuorum::HealCandidates) {
|
||||
// Candidate presence is not mutation authority. Every disk must answer,
|
||||
// and the heal owner validates the committed generation before recreation.
|
||||
1
|
||||
} else {
|
||||
topology_complete &= counts.values().all(|(count, _)| *count >= write_quorum);
|
||||
required
|
||||
};
|
||||
let mut buckets = counts
|
||||
.into_values()
|
||||
.filter_map(|(count, bucket)| (count >= required).then_some(bucket))
|
||||
.filter_map(|(count, bucket)| (count >= minimum_presence).then_some(bucket))
|
||||
.collect::<Vec<_>>();
|
||||
buckets.sort_by(|left, right| left.name.cmp(&right.name));
|
||||
Ok((buckets, topology_complete))
|
||||
|
||||
@@ -63,7 +63,6 @@ use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::disk::DiskAPI;
|
||||
#[cfg(test)]
|
||||
use crate::disk::DiskOption;
|
||||
#[cfg(test)]
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
#[cfg(test)]
|
||||
use crate::disk::new_disk;
|
||||
@@ -231,6 +230,60 @@ pub(crate) struct StaleMultipartCleanupGuard {
|
||||
lock_guard: ObjectLockDiagGuard,
|
||||
}
|
||||
|
||||
/// Exact-key recovery is admitted only after every disk proves that the upload
|
||||
/// metadata is absent. Keep the checked disk generation and upload lock through
|
||||
/// deletion; a quorum miss is not proof that an upload is an orphan.
|
||||
/// Final object data, completion replay identity, and old-data cleanup receipts
|
||||
/// live under the destination bucket/object, never this staging path. This also
|
||||
/// permits removing an exact orphan from a previous bucket incarnation without
|
||||
/// claiming that its UploadID belongs to the current incarnation.
|
||||
pub(crate) struct OrphanMultipartCleanupGuard {
|
||||
set: Arc<SetDisks>,
|
||||
disks: Vec<Option<DiskStore>>,
|
||||
upload_path: String,
|
||||
lock_guard: Arc<ObjectLockDiagGuard>,
|
||||
exists: bool,
|
||||
}
|
||||
|
||||
impl OrphanMultipartCleanupGuard {
|
||||
pub(crate) async fn delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<bool> {
|
||||
fence_commit_on_lock_loss(Some(&self.lock_guard), "orphan_multipart_cleanup", &self.upload_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
if !self.exists {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let results = join_all(self.disks.iter().map(|disk| async {
|
||||
let Some(disk) = disk else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
match disk
|
||||
.delete(
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&self.upload_path,
|
||||
crate::disk::DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Err(DiskError::FileNotFound | DiskError::PathNotFound | DiskError::VolumeNotFound) => Ok(()),
|
||||
result => result,
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
self.set.record_capacity_scope_if_needed(None, &self.disks);
|
||||
let errors = results.into_iter().map(|result| result.err()).collect::<Vec<_>>();
|
||||
if let Some(err) = reduce_write_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, self.disks.len()) {
|
||||
return Err(to_object_err(err.into(), vec![bucket, object]));
|
||||
}
|
||||
fence_commit_on_lock_loss(Some(&self.lock_guard), "orphan_multipart_cleanup", &self.upload_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct DataMovementMultipartAbortGuard {
|
||||
upload_path: String,
|
||||
write_quorum: Option<usize>,
|
||||
@@ -557,6 +610,34 @@ fn multipart_bucket_incarnation_matches(metadata: &HashMap<String, String>, expe
|
||||
matches!(multipart_bucket_incarnation_id(metadata), Ok(Some(actual)) if actual == expected)
|
||||
}
|
||||
|
||||
fn orphan_multipart_upload_id_is_canonical(upload_id: &str) -> bool {
|
||||
let Ok(decoded) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes()) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(decoded) = std::str::from_utf8(&decoded) else {
|
||||
return false;
|
||||
};
|
||||
let Some((_, raw)) = decoded.split_once('.') else {
|
||||
return false;
|
||||
};
|
||||
let Some((uuid, timestamp)) = raw.split_once('x') else {
|
||||
return false;
|
||||
};
|
||||
let Ok(id) = Uuid::parse_str(uuid) else {
|
||||
return false;
|
||||
};
|
||||
if id.is_nil() || id.to_string() != uuid {
|
||||
return false;
|
||||
}
|
||||
let Ok(timestamp_ns) = timestamp.parse::<i128>() else {
|
||||
return false;
|
||||
};
|
||||
if timestamp_ns.to_string() != timestamp {
|
||||
return false;
|
||||
}
|
||||
OffsetDateTime::from_unix_timestamp_nanos(timestamp_ns).is_ok()
|
||||
}
|
||||
|
||||
fn validate_multipart_bucket_incarnation(
|
||||
metadata: &HashMap<String, String>,
|
||||
bucket: &str,
|
||||
@@ -1246,6 +1327,114 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn lock_orphan_multipart_cleanup(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
opts: &ObjectOptions,
|
||||
held_guards: &[OrphanMultipartCleanupGuard],
|
||||
) -> Result<OrphanMultipartCleanupGuard> {
|
||||
let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
|
||||
if opts.data_movement
|
||||
|| opts.no_lock
|
||||
|| self.local_lock_manager.is_disabled()
|
||||
|| !orphan_multipart_upload_id_is_canonical(upload_id)
|
||||
{
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
// This path is derived from the caller's exact bucket/key and a canonical
|
||||
// UUID/timestamp upload identity, never from the shared MPU directory list.
|
||||
let upload_path = Self::get_upload_id_dir(bucket, object, upload_id);
|
||||
// Pool labels do not partition the raw ObjectKey sent to lock clients.
|
||||
// Reuse the actual owner only for this exact path and proven backend;
|
||||
// independent distributed domains still need their own guards.
|
||||
let distributed = self.ctx.is_dist_erasure().await;
|
||||
let mut lock_guard = None;
|
||||
for held in held_guards {
|
||||
if held.upload_path != upload_path {
|
||||
continue;
|
||||
}
|
||||
let shared_domain = match (distributed, held.set.ctx.is_dist_erasure().await) {
|
||||
(false, false) => self.shares_namespace_lock_domain(&held.set).await,
|
||||
(true, true) => {
|
||||
let same = super::super::same_distributed_lock_domain(&self.lockers, &held.set.lockers);
|
||||
if !same
|
||||
&& self
|
||||
.lockers
|
||||
.iter()
|
||||
.any(|client| held.set.lockers.iter().any(|held_client| Arc::ptr_eq(client, held_client)))
|
||||
{
|
||||
// A guard for only part of another quorum is not proof
|
||||
// of ownership there, and acquiring it again may wait
|
||||
// on ourselves. Preserve all staging in this topology.
|
||||
return Err(Error::other("orphan multipart cleanup cannot cover partially overlapping lock domains"));
|
||||
}
|
||||
same
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if shared_domain {
|
||||
lock_guard = Some(Arc::clone(&held.lock_guard));
|
||||
break;
|
||||
}
|
||||
}
|
||||
let lock_guard = match lock_guard {
|
||||
Some(guard) => guard,
|
||||
None => Arc::new(
|
||||
self.acquire_write_lock_diag("orphan_multipart_cleanup", RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await?,
|
||||
),
|
||||
};
|
||||
fence_commit_on_lock_loss(Some(&lock_guard), "orphan_multipart_cleanup", &upload_path)?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.len() != self.set_drive_count || disks.is_empty() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
// Raw full fan-out cannot quorum-short-circuit or interpret an empty
|
||||
// version list as absence of the physical metadata file.
|
||||
let (_, errors) = Self::read_all_raw_file_info(&disks, RUSTFS_META_MULTIPART_BUCKET, &upload_path, false).await;
|
||||
if errors.len() != disks.len() {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
for error in errors {
|
||||
match error {
|
||||
Some(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
|
||||
// Even one valid metadata copy may still be recoverable. Never
|
||||
// convert a partial, corrupt, or unreachable upload into debris.
|
||||
None => return Err(invalid_upload()),
|
||||
Some(error) => return Err(to_object_err(error.into(), vec![bucket, object, upload_id])),
|
||||
}
|
||||
}
|
||||
let mut exists = false;
|
||||
for disk in &disks {
|
||||
let Some(disk) = disk else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
};
|
||||
match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await {
|
||||
Ok(entries) => {
|
||||
// Raw metadata reads report a directory named xl.meta as
|
||||
// FileNotFound. Its physical presence is unknown/corrupt
|
||||
// metadata, not proof that this staging can be deleted.
|
||||
if entries.iter().any(|entry| entry.trim_end_matches('/') == STORAGE_FORMAT_FILE) {
|
||||
return Err(to_object_err(DiskError::FileCorrupt.into(), vec![bucket, object, upload_id]));
|
||||
}
|
||||
exists = true;
|
||||
}
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => {}
|
||||
Err(error) => return Err(to_object_err(error.into(), vec![bucket, object, upload_id])),
|
||||
}
|
||||
}
|
||||
Ok(OrphanMultipartCleanupGuard {
|
||||
set: self.clone(),
|
||||
disks,
|
||||
upload_path,
|
||||
lock_guard,
|
||||
exists,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||
&self,
|
||||
@@ -1678,6 +1867,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
drop(admission_guard);
|
||||
}
|
||||
|
||||
let mut commit_owns_cleanup = false;
|
||||
let result: Result<PartInfo> = async {
|
||||
let erasure =
|
||||
Arc::new(coding::Erasure::try_new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size)
|
||||
@@ -1881,120 +2071,171 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.await?;
|
||||
(Some(upload_guard), Some(part_guard))
|
||||
};
|
||||
let (commit_fi, _) = self
|
||||
.check_upload_id_exists_with_opts(bucket, object, upload_id, multipart_size_limit.is_some(), opts)
|
||||
.await?;
|
||||
let commit_size_limit = multipart_size_limit_from_metadata(&commit_fi.metadata)?;
|
||||
if commit_size_limit != multipart_size_limit {
|
||||
return Err(Error::InvalidArgument(
|
||||
"multipart upload".to_string(),
|
||||
SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
|
||||
"size limit metadata changed or is missing".to_string(),
|
||||
));
|
||||
}
|
||||
ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
|
||||
ensure_multipart_bucket_incarnation(
|
||||
&self.ctx,
|
||||
&commit_fi,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
|
||||
fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
|
||||
fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_outer_lock",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
if let Some(limit) = commit_size_limit {
|
||||
let current_size = self
|
||||
.current_multipart_logical_size(bucket, object, upload_id, &upload_id_path, &commit_fi, part_id)
|
||||
.await?;
|
||||
let candidate_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
admitted_multipart_size(current_size, candidate_size, limit)?;
|
||||
}
|
||||
|
||||
let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(
|
||||
&[self.pool_index],
|
||||
DecommissionCapacityAdmission::ExistingMultipart,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await;
|
||||
if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_decommission_capacity",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
let rename_result = self
|
||||
.rename_part(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
part_info_buff.into(),
|
||||
write_quorum,
|
||||
Some(MultipartWriteQuorumContext {
|
||||
stage: MULTIPART_WRITE_QUORUM_RENAME_PART,
|
||||
// Only guards owned by this operation may outlive its caller. Internal
|
||||
// data-movement and borrowed-lock callers retain their synchronous
|
||||
// lifetime; a fence by itself cannot transfer the outer lock lease.
|
||||
let detach_commit_owner = _upload_commit_guard.is_some()
|
||||
&& opts.namespace_lock_fence.is_none()
|
||||
&& (opts.bucket_lifecycle_lock_fence.is_none() || opts.bucket_lifecycle_lock_owner.is_some());
|
||||
let commit_set = self.clone();
|
||||
let commit_bucket = bucket.to_owned();
|
||||
let commit_object = object.to_owned();
|
||||
let commit_upload_id = upload_id.to_owned();
|
||||
let commit_upload_id_path = upload_id_path.clone();
|
||||
let commit_tmp_part = tmp_part.clone();
|
||||
let commit_opts = opts.clone();
|
||||
commit_owns_cleanup = true;
|
||||
// The body is already staged. Keep every commit guard through the
|
||||
// prepare, publish, settlement/rollback and temporary cleanup I/O,
|
||||
// even when the HTTP request stops awaiting this task.
|
||||
let commit = async move {
|
||||
let _upload_commit_guard = _upload_commit_guard;
|
||||
let _part_commit_guard = _part_commit_guard;
|
||||
let _capped_staging_guard = _capped_staging_guard;
|
||||
let bucket = commit_bucket.as_str();
|
||||
let object = commit_object.as_str();
|
||||
let upload_id = commit_upload_id.as_str();
|
||||
let upload_id_path = commit_upload_id_path;
|
||||
let opts = &commit_opts;
|
||||
let result: Result<PartInfo> = async {
|
||||
let (commit_fi, _) = commit_set
|
||||
.check_upload_id_exists_with_opts(bucket, object, upload_id, multipart_size_limit.is_some(), opts)
|
||||
.await?;
|
||||
let commit_size_limit = multipart_size_limit_from_metadata(&commit_fi.metadata)?;
|
||||
if commit_size_limit != multipart_size_limit {
|
||||
return Err(Error::InvalidArgument(
|
||||
"multipart upload".to_string(),
|
||||
SUFFIX_MAX_TOTAL_OBJECT_SIZE.to_string(),
|
||||
"size limit metadata changed or is missing".to_string(),
|
||||
));
|
||||
}
|
||||
ensure_data_movement_upload_access(&commit_fi, bucket, object, upload_id, opts)?;
|
||||
ensure_multipart_bucket_incarnation(
|
||||
&commit_set.ctx,
|
||||
&commit_fi,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
part_number: Some(part_id),
|
||||
}),
|
||||
)
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost).await;
|
||||
fence_commit_on_lock_loss(_upload_commit_guard.as_ref(), "put_object_part_commit", &upload_id_path)?;
|
||||
fence_commit_on_lock_loss(_part_commit_guard.as_ref(), "put_object_part_commit", &part_lock_path)?;
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_outer_lock",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
if let Some(limit) = commit_size_limit {
|
||||
let current_size = commit_set
|
||||
.current_multipart_logical_size(bucket, object, upload_id, &upload_id_path, &commit_fi, part_id)
|
||||
.await?;
|
||||
let candidate_size = u64::try_from(actual_size).map_err(|_| Error::PartMissingOrCorrupt)?;
|
||||
admitted_multipart_size(current_size, candidate_size, limit)?;
|
||||
}
|
||||
|
||||
let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(
|
||||
&[commit_set.pool_index],
|
||||
DecommissionCapacityAdmission::ExistingMultipart,
|
||||
)
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await;
|
||||
if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_decommission_capacity",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
let rename_result = commit_set
|
||||
.rename_part(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
&tmp_part_path,
|
||||
RUSTFS_META_MULTIPART_BUCKET,
|
||||
&part_path,
|
||||
part_info_buff.into(),
|
||||
write_quorum,
|
||||
Some(MultipartWriteQuorumContext {
|
||||
stage: MULTIPART_WRITE_QUORUM_RENAME_PART,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
part_number: Some(part_id),
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
drop(decommission_capacity_guard);
|
||||
let _ = rename_result?;
|
||||
#[cfg(test)]
|
||||
observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
|
||||
|
||||
let ret: PartInfo = PartInfo {
|
||||
etag: Some(etag.clone()),
|
||||
part_num: part_id,
|
||||
last_mod: Some(OffsetDateTime::now_utc()),
|
||||
size: w_size,
|
||||
actual_size,
|
||||
};
|
||||
|
||||
// error!("put_object_part ret {:?}", &ret);
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
.await;
|
||||
drop(decommission_capacity_guard);
|
||||
let _ = rename_result?;
|
||||
#[cfg(test)]
|
||||
observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterRename).await;
|
||||
drop(_part_commit_guard);
|
||||
drop(_upload_commit_guard);
|
||||
|
||||
let ret: PartInfo = PartInfo {
|
||||
etag: Some(etag.clone()),
|
||||
part_num: part_id,
|
||||
last_mod: Some(OffsetDateTime::now_utc()),
|
||||
size: w_size,
|
||||
actual_size,
|
||||
if result.is_err()
|
||||
&& let Err(err) = commit_set.delete_all(RUSTFS_META_TMP_BUCKET, &commit_tmp_part).await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_MULTIPART,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
result = "cleanup_failed",
|
||||
bucket,
|
||||
object,
|
||||
error = ?err,
|
||||
"failed to clean multipart commit staging"
|
||||
);
|
||||
}
|
||||
result
|
||||
};
|
||||
|
||||
// error!("put_object_part ret {:?}", &ret);
|
||||
|
||||
Ok(ret)
|
||||
if detach_commit_owner {
|
||||
tokio::spawn(commit)
|
||||
.await
|
||||
.map_err(|err| Error::other_with_context("put_object_part commit task failed", err))?
|
||||
} else {
|
||||
commit.await
|
||||
}
|
||||
}
|
||||
.await;
|
||||
|
||||
if result.is_err()
|
||||
if !commit_owns_cleanup
|
||||
&& result.is_err()
|
||||
&& let Err(err) = self.delete_all(RUSTFS_META_TMP_BUCKET, &tmp_part).await
|
||||
{
|
||||
warn!(tmp_part = %tmp_part, error = ?err, "failed to cleanup multipart temporary data");
|
||||
@@ -4233,6 +4474,300 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_multipart_identity_requires_canonical_upload_id() {
|
||||
let created = OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(10);
|
||||
let id = Uuid::new_v4();
|
||||
let timestamp = (created + time::Duration::seconds(1)).unix_timestamp_nanos();
|
||||
let encode = |raw: &str| runtime_sources::deployment_upload_id(raw);
|
||||
let upload_id = encode(&format!("{id}x{timestamp}"));
|
||||
assert!(orphan_multipart_upload_id_is_canonical(&upload_id));
|
||||
for raw in [
|
||||
format!("{}x{timestamp}", Uuid::nil()),
|
||||
format!("{}x{timestamp}", id.simple()),
|
||||
format!("{id}x0{timestamp}"),
|
||||
format!("{id}x+{timestamp}"),
|
||||
format!("{id}x{timestamp}/../another-upload"),
|
||||
format!("../{id}x{timestamp}"),
|
||||
format!("{id}x{}", i128::MAX),
|
||||
] {
|
||||
assert!(
|
||||
!orphan_multipart_upload_id_is_canonical(&encode(&raw)),
|
||||
"noncanonical upload identity must be rejected: {raw}"
|
||||
);
|
||||
}
|
||||
assert!(!orphan_multipart_upload_id_is_canonical("not-base64"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_requires_every_disk_to_prove_metadata_absence() {
|
||||
let (_temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "orphan-multipart-proof";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
|
||||
let metadata_path = format!("{upload_path}/{STORAGE_FORMAT_FILE}");
|
||||
let leftover_path = format!("{upload_path}/part.late");
|
||||
for disk in &disks {
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path, Bytes::from_static(b"late part"))
|
||||
.await
|
||||
.expect("write late part fixture");
|
||||
}
|
||||
for disk in &disks[..3] {
|
||||
disk.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
|
||||
.await
|
||||
.expect("remove majority metadata");
|
||||
}
|
||||
let opts = ObjectOptions::default();
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"one surviving metadata copy must block orphan recovery"
|
||||
);
|
||||
disks[3]
|
||||
.write_all(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Bytes::from_static(b"corrupt metadata"))
|
||||
.await
|
||||
.expect("corrupt final metadata copy");
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"corrupt metadata must not be treated as absence"
|
||||
);
|
||||
disks[3]
|
||||
.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
|
||||
.await
|
||||
.expect("remove final metadata copy");
|
||||
set.disks.write().await[3] = None;
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"an unreachable disk must not be hidden by absence quorum"
|
||||
);
|
||||
set.disks.write().await[3] = Some(disks[3].clone());
|
||||
for disk in &disks {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path)
|
||||
.await
|
||||
.expect("failed preflight must preserve every part"),
|
||||
Bytes::from_static(b"late part")
|
||||
);
|
||||
}
|
||||
|
||||
let guard = set
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.expect("all disks now prove metadata absence");
|
||||
assert!(guard.delete(bucket, object, &opts).await.expect("recover exact staging"));
|
||||
drop(guard);
|
||||
for disk in &disks {
|
||||
assert!(matches!(
|
||||
disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound)
|
||||
));
|
||||
}
|
||||
let guard = set
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &opts, &[])
|
||||
.await
|
||||
.expect("already absent staging remains provably absent");
|
||||
assert!(
|
||||
!guard
|
||||
.delete(bucket, object, &opts)
|
||||
.await
|
||||
.expect("absence does not report recovery")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_rejects_metadata_directory_on_one_disk() {
|
||||
let (temp_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let bucket = "orphan-multipart-directory";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disks, bucket).await;
|
||||
let upload = set
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload");
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload.upload_id);
|
||||
let metadata_path = format!("{upload_path}/{STORAGE_FORMAT_FILE}");
|
||||
let leftover_path = format!("{upload_path}/part.late");
|
||||
for disk in &disks {
|
||||
disk.delete(RUSTFS_META_MULTIPART_BUCKET, &metadata_path, Default::default())
|
||||
.await
|
||||
.expect("remove upload metadata");
|
||||
disk.write_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path, Bytes::from_static(b"late part"))
|
||||
.await
|
||||
.expect("write leftover part");
|
||||
}
|
||||
let metadata_directory = temp_dirs[3].path().join(RUSTFS_META_MULTIPART_BUCKET).join(&metadata_path);
|
||||
tokio::fs::create_dir_all(&metadata_directory)
|
||||
.await
|
||||
.expect("replace one missing metadata file with a directory");
|
||||
let (_, errors) =
|
||||
SetDisks::read_all_raw_file_info(&set.disks.read().await, RUSTFS_META_MULTIPART_BUCKET, &upload_path, false).await;
|
||||
assert!(
|
||||
errors.iter().all(|error| matches!(error, Some(DiskError::FileNotFound))),
|
||||
"raw metadata reads alone must reproduce the directory-as-absence case: {errors:?}"
|
||||
);
|
||||
assert!(
|
||||
set.lock_orphan_multipart_cleanup(bucket, object, &upload.upload_id, &ObjectOptions::default(), &[])
|
||||
.await
|
||||
.is_err(),
|
||||
"one physical metadata directory must block cleanup on every disk"
|
||||
);
|
||||
assert!(metadata_directory.is_dir(), "failed preflight must preserve the metadata directory");
|
||||
for disk in &disks {
|
||||
assert_eq!(
|
||||
disk.read_all(RUSTFS_META_MULTIPART_BUCKET, &leftover_path)
|
||||
.await
|
||||
.expect("failed preflight must preserve every leftover part"),
|
||||
Bytes::from_static(b"late part")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum OrphanMultipartLockDomainCase {
|
||||
Shared,
|
||||
Independent,
|
||||
PartiallyOverlapping,
|
||||
}
|
||||
|
||||
async fn check_orphan_multipart_distributed_lock_owners(case: OrphanMultipartLockDomainCase) {
|
||||
let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let first_client: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let second_client: Arc<dyn LockClient> =
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::new())));
|
||||
let second_clients = match case {
|
||||
OrphanMultipartLockDomainCase::Shared => vec![first_client.clone()],
|
||||
OrphanMultipartLockDomainCase::Independent => vec![second_client],
|
||||
OrphanMultipartLockDomainCase::PartiallyOverlapping => vec![first_client.clone(), second_client],
|
||||
};
|
||||
let (first_dirs, first_disks, first_set) =
|
||||
hermetic_set_disks_with_lockers_and_ctx(4, 0, 2, vec![first_client], ctx.clone()).await;
|
||||
let (second_dirs, second_disks, second_set) = hermetic_set_disks_with_lockers_and_ctx(4, 1, 2, second_clients, ctx).await;
|
||||
let bucket = "orphan-multipart-dist-locks";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&first_disks, bucket).await;
|
||||
make_bucket_on_all(&second_disks, bucket).await;
|
||||
let upload_id = runtime_sources::deployment_upload_id(&format!(
|
||||
"{}x{}",
|
||||
Uuid::new_v4(),
|
||||
OffsetDateTime::now_utc().unix_timestamp_nanos()
|
||||
));
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
for directory in first_dirs.iter().chain(second_dirs.iter()) {
|
||||
let staging = directory.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&upload_path);
|
||||
tokio::fs::create_dir_all(&staging)
|
||||
.await
|
||||
.expect("create exact orphan staging");
|
||||
tokio::fs::write(staging.join("part.late"), b"late part")
|
||||
.await
|
||||
.expect("write orphan part");
|
||||
}
|
||||
let opts = ObjectOptions::default();
|
||||
let first_guard = first_set
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &upload_id, &opts, &[])
|
||||
.await
|
||||
.expect("first domain has an orphan");
|
||||
let second_result = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
second_set.lock_orphan_multipart_cleanup(bucket, object, &upload_id, &opts, std::slice::from_ref(&first_guard)),
|
||||
)
|
||||
.await
|
||||
.expect("lock-domain admission must not wait on a guard it already owns");
|
||||
if case == OrphanMultipartLockDomainCase::PartiallyOverlapping {
|
||||
let error = match second_result {
|
||||
Ok(_) => panic!("one existing quorum cannot authorize a partially overlapping domain"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert!(error.to_string().contains("partially overlapping lock domains"));
|
||||
for directory in first_dirs.iter().chain(second_dirs.iter()) {
|
||||
assert!(
|
||||
directory
|
||||
.path()
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_path)
|
||||
.join("part.late")
|
||||
.exists(),
|
||||
"uncertain lock coverage must preserve every physical part"
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let second_guard = second_result.expect("the second pool has a proven lock domain");
|
||||
assert_eq!(
|
||||
Arc::ptr_eq(&first_guard.lock_guard, &second_guard.lock_guard),
|
||||
case == OrphanMultipartLockDomainCase::Shared,
|
||||
"only the same exact backend may share a real upload guard across pool labels"
|
||||
);
|
||||
let second_namespace = second_set
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut second_writer = Box::pin(second_namespace.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(futures::poll!(&mut second_writer).is_pending());
|
||||
if case == OrphanMultipartLockDomainCase::Shared {
|
||||
drop(first_guard);
|
||||
assert!(
|
||||
futures::poll!(&mut second_writer).is_pending(),
|
||||
"the second pool must retain the shared real owner after the first guard drops"
|
||||
);
|
||||
drop(second_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), second_writer)
|
||||
.await
|
||||
.expect("the shared backend should release after the last owner")
|
||||
.expect("shared-domain writer resumes"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let first_namespace = first_set
|
||||
.new_ns_lock(RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut first_writer = Box::pin(first_namespace.get_write_lock(Duration::from_secs(10)));
|
||||
assert!(futures::poll!(&mut first_writer).is_pending());
|
||||
drop(first_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), first_writer)
|
||||
.await
|
||||
.expect("first domain should release independently")
|
||||
.expect("first writer resumes"),
|
||||
);
|
||||
assert!(futures::poll!(&mut second_writer).is_pending(), "the second real owner must remain held");
|
||||
drop(second_guard);
|
||||
drop(
|
||||
tokio::time::timeout(Duration::from_secs(10), second_writer)
|
||||
.await
|
||||
.expect("second domain should release with its owner")
|
||||
.expect("second writer resumes"),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_keeps_separate_distributed_lock_owners() {
|
||||
check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::Independent).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_reuses_shared_distributed_owner_across_pools() {
|
||||
check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::Shared).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn orphan_multipart_cleanup_rejects_partially_overlapping_distributed_domains() {
|
||||
check_orphan_multipart_distributed_lock_owners(OrphanMultipartLockDomainCase::PartiallyOverlapping).await;
|
||||
}
|
||||
|
||||
async fn stage_upload_with_create_opts(
|
||||
set_disks: &Arc<SetDisks>,
|
||||
bucket: &str,
|
||||
@@ -6370,6 +6905,76 @@ mod tests {
|
||||
.expect("abort should delete the upload after UploadPart releases the lock");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn cancelled_put_object_part_keeps_upload_lock_through_commit() {
|
||||
for pause in [
|
||||
MultipartCommitPause::PutPartAfterCapacityAdmission,
|
||||
MultipartCommitPause::PutPartAfterRename,
|
||||
] {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
|
||||
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
|
||||
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
|
||||
let bucket = "multipart-cancelled-part-commit-bucket";
|
||||
let object = "object";
|
||||
make_bucket_on_all(&disk_stores, bucket).await;
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let upload_id = upload.upload_id;
|
||||
let upload_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
|
||||
signaling.set_target(rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_path.clone()));
|
||||
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
|
||||
let barrier = MultipartCommitBarrier::install(bucket, object, pause);
|
||||
|
||||
let put_store = set_disks.clone();
|
||||
let put_upload_id = upload_id.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x45; 4096]);
|
||||
put_store
|
||||
.put_object_part(bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
put.abort();
|
||||
assert!(
|
||||
put.await
|
||||
.expect_err("the request waiter should remain cancellable")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let abort_store = set_disks.clone();
|
||||
let abort_upload_id = upload_id.clone();
|
||||
let mut abort = tokio::spawn(async move {
|
||||
abort_store
|
||||
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
signaling.wait_for_attempts(2).await;
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut abort).await.is_err(),
|
||||
"Abort must not acquire the upload lock while a cancelled request's commit is paused at {pause:?}"
|
||||
);
|
||||
barrier.release();
|
||||
tokio::time::timeout(Duration::from_secs(10), abort)
|
||||
.await
|
||||
.expect("Abort should finish after the commit drains")
|
||||
.expect("Abort task should not panic")
|
||||
.expect("Abort should delete the committed part staging");
|
||||
for disk in &disk_stores {
|
||||
assert!(
|
||||
matches!(
|
||||
disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, -1).await,
|
||||
Err(DiskError::FileNotFound)
|
||||
),
|
||||
"the upload directory must be absent after the commit and Abort finish"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn put_object_part_different_part_numbers_commit_concurrently() {
|
||||
@@ -9799,6 +10404,18 @@ mod tests {
|
||||
"the post-commit crash must leave the upload listable for reclamation"
|
||||
);
|
||||
|
||||
let cleanup = set_disks
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &u_new, &ObjectOptions::default(), &[])
|
||||
.await
|
||||
.expect("the committed upload has no staging metadata on any disk");
|
||||
assert!(
|
||||
cleanup
|
||||
.delete(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recover completed upload staging")
|
||||
);
|
||||
drop(cleanup);
|
||||
|
||||
// The commit rename already consumed the upload's xl.meta, so the
|
||||
// retry cannot read staging. It must still return the committed
|
||||
// object: AWS CompleteMultipartUpload is idempotent for the same
|
||||
@@ -9817,21 +10434,6 @@ mod tests {
|
||||
!upload_is_listed(&set_disks, bucket, object, &u_new).await,
|
||||
"the idempotent retry must reclaim staging left behind by the crash"
|
||||
);
|
||||
|
||||
// Reclaim the leftover exactly as the production tail does: delete_all
|
||||
// on the upload path (abort_multipart_upload cannot — the upload's
|
||||
// metadata is gone, so it too answers InvalidUploadID).
|
||||
let upload_dir = SetDisks::get_upload_id_dir(bucket, object, &u_new);
|
||||
set_disks
|
||||
.delete_all(RUSTFS_META_MULTIPART_BUCKET, &upload_dir)
|
||||
.await
|
||||
.expect("sweeping the leftover upload staging should succeed");
|
||||
assert!(
|
||||
!upload_is_listed(&set_disks, bucket, object, &u_new).await,
|
||||
"reclaiming the upload must drop it from the listing"
|
||||
);
|
||||
let (body_after, _) = read_object(&set_disks, bucket, object).await;
|
||||
assert_eq!(body_after, new, "reclaiming the leftover upload must not disturb the committed object");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -9878,6 +10480,18 @@ mod tests {
|
||||
.expect("the failed post-commit completion should release its object guard"),
|
||||
);
|
||||
|
||||
let cleanup = set_disks
|
||||
.lock_orphan_multipart_cleanup(bucket, object, &u_new, &ObjectOptions::default(), &[])
|
||||
.await
|
||||
.expect("committed staging has no upload metadata");
|
||||
assert!(
|
||||
cleanup
|
||||
.delete(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("orphan recovery must only remove upload staging")
|
||||
);
|
||||
drop(cleanup);
|
||||
|
||||
let (body, _) = read_object(&set_disks, bucket, object).await;
|
||||
assert_eq!(body, new, "the committed replacement must remain readable after the crash");
|
||||
let mut receipts = 0;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,9 +15,9 @@
|
||||
use super::{
|
||||
Arc, Bytes, DiskError, DiskOption, DiskStore, Endpoint, ErasureCache, Error, FileInfo, GetCodecStreamingFallbackReason,
|
||||
GetObjectFileInfo, GetObjectMetadataCacheCapability, GetObjectMetadataCacheEntry, GetObjectMetadataCacheGeneration,
|
||||
GetObjectMetadataCacheKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK,
|
||||
OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks, StorageError,
|
||||
adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs,
|
||||
GetObjectMetadataCacheKey, GetObjectMetadataCacheLookupKey, GetObjectReadPolicy, HashAlgorithm, LOG_COMPONENT_ECSTORE,
|
||||
LOG_SUBSYSTEM_SET_DISK, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectOptions, RUSTFS_META_BUCKET, ReadOptions, Result, SetDisks,
|
||||
StorageError, adaptive_duplex_buffer_size, build_get_codec_streaming_decode_engine, build_inline_bitrot_readers_from_refs,
|
||||
collect_inline_data_shard_fileinfos_by_index, debug, error, get_codec_streaming_metrics_path,
|
||||
get_codec_streaming_multipart_max_parts, get_object_read_policy, is_codec_streaming_multipart_enabled,
|
||||
is_multipart_reader_setup_prefetch_enabled, object_fits_single_block, reduce_read_quorum_errs, to_object_err,
|
||||
@@ -208,6 +208,16 @@ use tokio::time::timeout;
|
||||
#[cfg(test)]
|
||||
use uuid::Uuid;
|
||||
|
||||
fn metadata_quorum_outcome(error: &StorageError) -> &'static str {
|
||||
if error.is_not_found() {
|
||||
"not_found"
|
||||
} else if error.is_quorum_error() {
|
||||
"not_reached"
|
||||
} else {
|
||||
"unavailable"
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct GetObjectDownstreamWriter<W> {
|
||||
inner: W,
|
||||
}
|
||||
@@ -400,7 +410,7 @@ impl SetDisks {
|
||||
let Some(generation) = self.get_object_metadata_cache_generation(bucket, object) else {
|
||||
return MetadataCacheLookup::Miss;
|
||||
};
|
||||
let key = GetObjectMetadataCacheKey::new(bucket, object, generation);
|
||||
let key = GetObjectMetadataCacheLookupKey::new(bucket, object, generation);
|
||||
// moka handles TTL expiry automatically; no is_fresh() check needed
|
||||
let Some(entry) = self.get_object_metadata_cache.get(&key).await else {
|
||||
return MetadataCacheLookup::Miss;
|
||||
@@ -462,14 +472,14 @@ impl SetDisks {
|
||||
entry: Arc<GetObjectMetadataCacheEntry>,
|
||||
after_insert: impl FnOnce(),
|
||||
) {
|
||||
// Every publisher shares Moka's per-key compute lock. A conditional
|
||||
// snapshot cannot downgrade a full layout or extend its original TTL.
|
||||
// Every publisher shares Moka's per-key compute lock. A partial snapshot
|
||||
// cannot downgrade a full layout or extend its original TTL.
|
||||
self.get_object_metadata_cache
|
||||
.entry(key.clone())
|
||||
.and_compute_with(|current| {
|
||||
std::future::ready(
|
||||
if !self.is_get_object_metadata_cache_generation_current(generation)
|
||||
|| (entry.capability == GetObjectMetadataCacheCapability::ConditionOnly
|
||||
|| (entry.capability != GetObjectMetadataCacheCapability::Full
|
||||
&& current
|
||||
.as_ref()
|
||||
.is_some_and(|existing| existing.value().capability == GetObjectMetadataCacheCapability::Full))
|
||||
@@ -602,6 +612,11 @@ impl SetDisks {
|
||||
purpose: MetadataReadPurpose,
|
||||
) -> Result<GetObjectFileInfo> {
|
||||
let vid = opts.version_id.clone().unwrap_or_default();
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX
|
||||
};
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
|
||||
let metadata_cache_lookup_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
@@ -617,12 +632,17 @@ impl SetDisks {
|
||||
match self.lookup_cached_get_object_fileinfo(bucket, object).await {
|
||||
MetadataCacheLookup::Hit(cached)
|
||||
if cached.capability == GetObjectMetadataCacheCapability::Full
|
||||
|| (cached.capability == GetObjectMetadataCacheCapability::QuorumSelected
|
||||
&& read_data
|
||||
&& allow_early_stop)
|
||||
|| (purpose == MetadataReadPurpose::GetObjectReader
|
||||
&& cached.capability == GetObjectMetadataCacheCapability::ConditionOnly
|
||||
&& allow_early_stop
|
||||
&& crate::object_api::get_object_read_condition_is_terminal(
|
||||
crate::object_api::GetObjectReadMetadata::from_file_info(bucket, object, &cached.fi),
|
||||
)) =>
|
||||
{
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "reached");
|
||||
rustfs_io_metrics::record_get_object_metadata_cache_decision(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
GET_METADATA_CACHE_DECISION_HIT,
|
||||
@@ -633,7 +653,15 @@ impl SetDisks {
|
||||
GET_STAGE_METADATA_CACHE_LOOKUP,
|
||||
metadata_cache_lookup_start,
|
||||
);
|
||||
return Ok(GetObjectFileInfo::shared(cached));
|
||||
let snapshot = if cached.capability == GetObjectMetadataCacheCapability::QuorumSelected
|
||||
&& non_inline_data_read_early_stop_allowed(read_data, bucket, object)
|
||||
&& late_materialization_candidate_is_safe(&cached.fi)
|
||||
{
|
||||
GetObjectFileInfo::shared_with_late_metadata_refresh(cached)
|
||||
} else {
|
||||
GetObjectFileInfo::shared(cached)
|
||||
};
|
||||
return Ok(snapshot);
|
||||
}
|
||||
MetadataCacheLookup::Hit(_) => {
|
||||
// Keep the conditional entry for other requests; this read
|
||||
@@ -683,7 +711,7 @@ impl SetDisks {
|
||||
let allow_read_version_coalescing = purpose == MetadataReadPurpose::GetObjectReader
|
||||
&& !crate::bucket::utils::is_meta_bucketname(bucket)
|
||||
&& crate::runtime::global::get_metadata_read_version_coalescing_service_ready();
|
||||
let metadata_read = if allow_read_version_coalescing {
|
||||
let metadata_read_result = if allow_read_version_coalescing {
|
||||
Self::read_metadata_for_get_object(
|
||||
&disks,
|
||||
"",
|
||||
@@ -695,7 +723,7 @@ impl SetDisks {
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?
|
||||
.await
|
||||
} else {
|
||||
Self::read_metadata_observed(
|
||||
&disks,
|
||||
@@ -709,16 +737,19 @@ impl SetDisks {
|
||||
allow_early_stop,
|
||||
self.default_parity_count,
|
||||
)
|
||||
.await?
|
||||
.await
|
||||
};
|
||||
let metadata_read = match metadata_read_result {
|
||||
Ok(metadata_read) => metadata_read,
|
||||
Err(err) => {
|
||||
let err = StorageError::from(err);
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&err));
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let metadata_fanout_complete = metadata_read.is_complete();
|
||||
let condition_stopped = metadata_read.condition_stopped;
|
||||
let (mut parts_metadata, errs, metadata_fanout_diagnostics) = metadata_read.into_legacy();
|
||||
let metadata_metrics_path = if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX
|
||||
};
|
||||
metadata_fanout_diagnostics.record(metadata_metrics_path);
|
||||
// warn!("get_object_fileinfo parts_metadata {:?}", &parts_metadata);
|
||||
// warn!("get_object_fileinfo {}/{} errs {:?}", bucket, object, &errs);
|
||||
@@ -731,6 +762,7 @@ impl SetDisks {
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, metadata_quorum_outcome(&e));
|
||||
// error!("Self::object_quorum_from_meta: {:?}, bucket: {}, object: {}", &e, bucket, object);
|
||||
record_get_stage_duration_if_enabled(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
@@ -745,6 +777,7 @@ impl SetDisks {
|
||||
let write_quorum = usize::try_from(write_quorum)
|
||||
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(metadata_metrics_path, "not_reached");
|
||||
error!("reduce_read_quorum_errs: {:?}, bucket: {}, object: {}", &err, bucket, object);
|
||||
record_get_stage_duration_if_enabled(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
@@ -755,7 +788,17 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let (op_online_disks, mut fi, fileinfo_selection_quorum) =
|
||||
Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum)?;
|
||||
match Self::select_valid_fileinfo(&disks, &parts_metadata, &errs, vid.as_str(), read_quorum, write_quorum) {
|
||||
Ok(selection) => selection,
|
||||
Err(err) => {
|
||||
let err = StorageError::from(err);
|
||||
rustfs_io_metrics::record_get_object_metadata_quorum_result(
|
||||
metadata_metrics_path,
|
||||
metadata_quorum_outcome(&err),
|
||||
);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
let include_part_checksums =
|
||||
opts.include_part_checksums || opts.part_number.is_some() || opts.data_movement || opts.raw_data_movement_read;
|
||||
if include_part_checksums {
|
||||
@@ -778,7 +821,16 @@ impl SetDisks {
|
||||
"metadata_read_error",
|
||||
)
|
||||
.await;
|
||||
} else if use_metadata_cache && (metadata_fanout_complete || condition_stopped) {
|
||||
} else if use_metadata_cache
|
||||
&& (metadata_fanout_complete
|
||||
|| condition_stopped
|
||||
|| (read_data
|
||||
&& allow_early_stop
|
||||
&& metadata_fanout_diagnostics.valid_responses() >= read_quorum
|
||||
&& errs.iter().all(Option::is_none)
|
||||
&& fileinfo_selection_quorum >= read_quorum
|
||||
&& op_online_disks.iter().filter(|disk| disk.is_some()).count() >= read_quorum))
|
||||
{
|
||||
#[cfg(test)]
|
||||
metadata_cache_tests::wait_before_metadata_cache_publish(bucket, object).await;
|
||||
self.cache_get_object_fileinfo(
|
||||
@@ -788,8 +840,10 @@ impl SetDisks {
|
||||
&fi,
|
||||
if metadata_fanout_complete {
|
||||
GetObjectMetadataCacheCapability::Full
|
||||
} else {
|
||||
} else if condition_stopped {
|
||||
GetObjectMetadataCacheCapability::ConditionOnly
|
||||
} else {
|
||||
GetObjectMetadataCacheCapability::QuorumSelected
|
||||
},
|
||||
),
|
||||
&parts_metadata,
|
||||
@@ -2650,6 +2704,7 @@ fn is_get_object_metadata_cache_request_eligible(bucket: &str, opts: &ObjectOpti
|
||||
#[cfg(test)]
|
||||
mod metadata_cache_tests {
|
||||
use super::*;
|
||||
use crate::set_disk::core::io_primitives::disk_call_counters;
|
||||
use rustfs_heal_contracts::heal_channel::HealAdmissionDropReason;
|
||||
use serial_test::serial;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
@@ -4009,7 +4064,7 @@ mod metadata_cache_tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(metadata_cache_publish_barrier)]
|
||||
#[serial(metadata_cache_early_stop)]
|
||||
async fn metadata_cache_production_fanout_cannot_publish_after_invalidation() {
|
||||
// Isolated context: an ambient DistErasure window (another test's
|
||||
// SetupTypeGuard) would bypass metadata-cache publication entirely
|
||||
@@ -4063,6 +4118,191 @@ mod metadata_cache_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial(metadata_cache_early_stop)]
|
||||
async fn metadata_cache_publishes_quorum_selected_early_stop_result() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
isolated_ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
|
||||
.await;
|
||||
let (dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await;
|
||||
let bucket = "metadata-cache-early-stop";
|
||||
let object = "objects/repeated-key";
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
disk.make_volume(bucket)
|
||||
.await
|
||||
.expect("test bucket should be created on each disk");
|
||||
}
|
||||
let expected_body = vec![0x5a; 1024 * 1024];
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(expected_body.clone()),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("test object should be written before enabling the slow-tail fault");
|
||||
|
||||
let calls = disk_call_counters::observe(object);
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let first = tokio::time::timeout(
|
||||
Duration::from_millis(400),
|
||||
set.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true),
|
||||
)
|
||||
.await
|
||||
.expect("first metadata read should stop before the delayed peer")
|
||||
.expect("read quorum should provide valid object metadata");
|
||||
assert!(first.owned.is_some(), "the first lookup should resolve through disk metadata");
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4);
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT), 1);
|
||||
|
||||
let second = set
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true)
|
||||
.await
|
||||
.expect("second lookup should use the quorum-published metadata cache entry");
|
||||
assert!(second.shared_entry().is_some(), "the repeated GET should hit the published cache entry");
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 4);
|
||||
|
||||
let cached = second.shared_entry().expect("second lookup should share cached metadata");
|
||||
assert_eq!(cached.fi.name, object);
|
||||
let cached_online_count = cached.online_disks.iter().filter(|disk| disk.is_some()).count();
|
||||
assert_eq!(cached_online_count, 3, "the warm cache should contain three of four disks");
|
||||
assert_eq!(cached.capability, GetObjectMetadataCacheCapability::QuorumSelected);
|
||||
|
||||
let data_dir = cached
|
||||
.fi
|
||||
.data_dir
|
||||
.expect("quorum cache retains the object data directory")
|
||||
.to_string();
|
||||
let selected_disk_indices = cached
|
||||
.online_disks
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, disk)| disk.as_ref().map(|_| index))
|
||||
.take(2)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(selected_disk_indices.len(), 2, "the warmed cache should name enough selected shards");
|
||||
for index in selected_disk_indices {
|
||||
tokio::fs::remove_file(dirs[index].path().join(bucket).join(object).join(&data_dir).join("part.1"))
|
||||
.await
|
||||
.expect("simulate loss of two cached selected shards");
|
||||
}
|
||||
|
||||
let mut reader = set
|
||||
.get_object_reader(bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("a cached partial layout must recover through omitted disks");
|
||||
let mut body = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.expect("the GET should decode after refreshing the omitted disks");
|
||||
assert_eq!(body, expected_body, "recovered body must be byte-exact");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let full_layout = set
|
||||
.get_object_fileinfo_gated(bucket, object, &ObjectOptions::default(), false, false)
|
||||
.await
|
||||
.expect("read-before-write consumers must fetch a complete layout");
|
||||
assert!(
|
||||
full_layout.shared_entry().is_none(),
|
||||
"a quorum-selected partial layout cannot satisfy mutation consumers"
|
||||
);
|
||||
assert!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION) > 4,
|
||||
"mutation consumers must independently fan out after a quorum-only cache hit"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial(metadata_cache_early_stop)]
|
||||
async fn metadata_cache_early_stop_publication_is_fenced_by_invalidation() {
|
||||
let isolated_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
isolated_ctx
|
||||
.update_erasure_type(crate::layout::endpoints::SetupType::Erasure)
|
||||
.await;
|
||||
let (_dirs, set) = crate::ecstore_validation_blackbox::make_local_set_disks_with_ctx(4, 2, isolated_ctx).await;
|
||||
let bucket = "metadata-cache-early-stop-fence";
|
||||
let object = "objects/repeated-key";
|
||||
let disks = set.disks.read().await.clone();
|
||||
for disk in disks.iter().flatten() {
|
||||
disk.make_volume(bucket)
|
||||
.await
|
||||
.expect("test bucket should be created on each disk");
|
||||
}
|
||||
set.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut PutObjReader::from_vec(vec![0x5a; 1024 * 1024]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("test object should be written before enabling the slow-tail fault");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("false")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS", Some("500")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS", Some("3")),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET", Some(bucket)),
|
||||
("RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX", Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let barrier = MetadataCachePublishBarrier::install(bucket, object);
|
||||
let stale_generation = set
|
||||
.get_object_metadata_cache_generation(bucket, object)
|
||||
.expect("metadata cache generation should be active");
|
||||
let reader_set = Arc::clone(&set);
|
||||
let read = tokio::spawn(async move {
|
||||
reader_set
|
||||
.get_object_fileinfo(bucket, object, &ObjectOptions::default(), true, true)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
set.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
barrier.release();
|
||||
|
||||
let snapshot = read
|
||||
.await
|
||||
.expect("early-stop metadata read should not panic")
|
||||
.expect("read quorum should resolve the selected FileInfo");
|
||||
assert!(snapshot.owned.is_some());
|
||||
assert!(
|
||||
set.get_object_metadata_cache
|
||||
.get(&GetObjectMetadataCacheKey::new(bucket, object, stale_generation))
|
||||
.await
|
||||
.is_none(),
|
||||
"early-stop publication must not retain an entry under the retired generation"
|
||||
);
|
||||
assert!(
|
||||
set.cached_get_object_fileinfo(bucket, object).await.is_none(),
|
||||
"invalidation must reject the quorum-selected early-stop publication"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial(metadata_cache_publish_barrier, body_cache_hook)]
|
||||
async fn conditional_inline_hedge_caches_only_terminal_metadata() {
|
||||
@@ -5082,12 +5322,15 @@ mod metadata_cache_tests {
|
||||
.get_object_metadata_cache_generation("bucket-a", "object-a")
|
||||
.expect("metadata cache generation should be active");
|
||||
let first_key = GetObjectMetadataCacheKey::new("bucket-a", "object-a", generation);
|
||||
let first_lookup = GetObjectMetadataCacheLookupKey::new("bucket-a", "object-a", generation);
|
||||
let second_key = GetObjectMetadataCacheKey {
|
||||
bucket: Arc::from("bucket-b"),
|
||||
object: Arc::from("object-b"),
|
||||
generation: generation.value,
|
||||
hash: generation.hash,
|
||||
};
|
||||
let second_lookup = GetObjectMetadataCacheLookupKey::new("bucket-b", "object-b", generation);
|
||||
let unrelated_lookup = GetObjectMetadataCacheLookupKey::new("bucket-c", "object-c", generation);
|
||||
let first_fi = valid_test_fileinfo("object-a");
|
||||
let second_fi = valid_test_fileinfo("object-b");
|
||||
let entry = |fi: FileInfo| {
|
||||
@@ -5108,7 +5351,7 @@ mod metadata_cache_tests {
|
||||
|
||||
assert_eq!(
|
||||
set.get_object_metadata_cache
|
||||
.get(&first_key)
|
||||
.get(&first_lookup)
|
||||
.await
|
||||
.expect("first colliding entry should remain addressable")
|
||||
.fi
|
||||
@@ -5117,13 +5360,14 @@ mod metadata_cache_tests {
|
||||
);
|
||||
assert_eq!(
|
||||
set.get_object_metadata_cache
|
||||
.get(&second_key)
|
||||
.get(&second_lookup)
|
||||
.await
|
||||
.expect("second colliding entry should remain addressable")
|
||||
.fi
|
||||
.name,
|
||||
"object-b"
|
||||
);
|
||||
assert!(set.get_object_metadata_cache.get(&unrelated_lookup).await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -5290,6 +5534,16 @@ mod tests {
|
||||
const CODEC_STREAMING_TEST_OBJECT: &str = "object";
|
||||
static CAPTURED_READ_REPAIR_CALLS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
#[test]
|
||||
fn metadata_quorum_outcome_uses_bounded_error_categories() {
|
||||
assert_eq!(metadata_quorum_outcome(&StorageError::ErasureReadQuorum), "not_reached");
|
||||
assert_eq!(metadata_quorum_outcome(&StorageError::FileNotFound), "not_found");
|
||||
assert_eq!(
|
||||
metadata_quorum_outcome(&StorageError::Io(std::io::Error::other("disk unavailable"))),
|
||||
"unavailable"
|
||||
);
|
||||
}
|
||||
|
||||
fn capture_read_repair_submitter(
|
||||
_request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
|
||||
) -> ReadRepairAdmissionFuture {
|
||||
|
||||
@@ -20,6 +20,7 @@ use crate::bucket::{
|
||||
use crate::error::is_err_bucket_not_found;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::set_disk::{BucketInfoQuorum, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::admin::DiskSetSelector;
|
||||
use crate::storage_api_contracts::bucket::{BUCKET_LIFECYCLE_LOCK_OBJECT, SRBucketDeleteOp};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use futures::stream::{self, StreamExt};
|
||||
@@ -1151,7 +1152,7 @@ impl ECStore {
|
||||
let mut scoped_results = futures::future::join_all(self.bucket_sets().map(|(pool_index, set_index, set)| async move {
|
||||
let result = match quorum {
|
||||
BucketInfoQuorum::Read => set.stat_bucket_with_quorum(bucket, quorum).await,
|
||||
BucketInfoQuorum::Write => set.get_bucket_info(bucket, opts).await,
|
||||
BucketInfoQuorum::Write | BucketInfoQuorum::HealCandidates => set.get_bucket_info(bucket, opts).await,
|
||||
};
|
||||
(pool_index, set_index, result)
|
||||
}))
|
||||
@@ -1232,13 +1233,13 @@ impl ECStore {
|
||||
pub(super) async fn handle_list_bucket(&self, opts: &BucketOptions) -> Result<Vec<BucketInfo>> {
|
||||
// TODO(backlog): support cached bucket listing via opts.cached
|
||||
Ok(self
|
||||
.list_bucket_from_sets(opts, usize::MAX, BucketInfoQuorum::Write)
|
||||
.list_bucket_from_sets(opts, usize::MAX, BucketInfoQuorum::Write, None)
|
||||
.await?
|
||||
.buckets)
|
||||
}
|
||||
|
||||
pub async fn list_bucket_for_scanner(&self, opts: &BucketOptions) -> Result<crate::cluster::rpc::ScannerBucketListing> {
|
||||
self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY, BucketInfoQuorum::Write)
|
||||
self.list_bucket_from_sets(opts, SCANNER_BUCKET_LIST_SET_CONCURRENCY, BucketInfoQuorum::Write, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1254,20 +1255,44 @@ impl ECStore {
|
||||
},
|
||||
SCANNER_BUCKET_LIST_SET_CONCURRENCY,
|
||||
BucketInfoQuorum::Read,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
.buckets)
|
||||
}
|
||||
|
||||
/// Preserve namespace completeness for administrator traversal. A scoped
|
||||
/// request must not depend on unrelated sets being available.
|
||||
pub async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
selector: Option<DiskSetSelector>,
|
||||
) -> Result<crate::cluster::rpc::ScannerBucketListing> {
|
||||
self.list_bucket_from_sets(
|
||||
&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
},
|
||||
SCANNER_BUCKET_LIST_SET_CONCURRENCY,
|
||||
BucketInfoQuorum::HealCandidates,
|
||||
selector,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_bucket_from_sets(
|
||||
&self,
|
||||
opts: &BucketOptions,
|
||||
max_concurrency: usize,
|
||||
quorum: BucketInfoQuorum,
|
||||
selector: Option<DiskSetSelector>,
|
||||
) -> Result<crate::cluster::rpc::ScannerBucketListing> {
|
||||
let set_count = self.pools.iter().map(|pool| pool.disk_set.len()).sum();
|
||||
let sets = self
|
||||
.bucket_sets()
|
||||
.filter(|(pool, set, _)| selector.is_none_or(|scope| scope.pool_idx == *pool && scope.set_idx == *set))
|
||||
.collect::<Vec<_>>();
|
||||
let set_count = sets.len();
|
||||
let concurrency = bucket_list_set_concurrency(set_count, max_concurrency);
|
||||
let mut set_listings = stream::iter(self.bucket_sets().map(move |(pool_index, set_index, set)| async move {
|
||||
let mut set_listings = stream::iter(sets.into_iter().map(move |(pool_index, set_index, set)| async move {
|
||||
set.list_bucket_with_quorum(quorum)
|
||||
.await
|
||||
.map(|(buckets, complete)| (pool_index, set_index, buckets, complete))
|
||||
@@ -2236,6 +2261,64 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_bucket_discovery_preserves_completeness_and_scope() {
|
||||
use crate::storage_api_contracts::admin::DiskSetSelector;
|
||||
let (temp, store) = setup_bucket_quorum_test_env(&[2, 2], None).await;
|
||||
let empty = store
|
||||
.list_buckets_for_admin_heal(None)
|
||||
.await
|
||||
.expect("empty namespace enumeration");
|
||||
assert!(empty.buckets.is_empty());
|
||||
assert!(empty.topology_complete);
|
||||
for disk in 0..2 {
|
||||
tokio::fs::create_dir(temp.path().join(format!("pool0-disk{disk}/healthy-bucket")))
|
||||
.await
|
||||
.expect("healthy bucket");
|
||||
}
|
||||
tokio::fs::create_dir(temp.path().join("pool1-disk0/partial-bucket"))
|
||||
.await
|
||||
.expect("single surviving bucket directory");
|
||||
let global = store
|
||||
.list_buckets_for_admin_heal(None)
|
||||
.await
|
||||
.expect("online disk enumeration");
|
||||
assert!(global.topology_complete, "every disk answered the candidate inventory");
|
||||
assert_eq!(
|
||||
global.buckets.iter().map(|b| b.name.as_str()).collect::<Vec<_>>(),
|
||||
["healthy-bucket", "partial-bucket"]
|
||||
);
|
||||
let healthy = store
|
||||
.list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 0)))
|
||||
.await
|
||||
.expect("healthy scope");
|
||||
assert!(healthy.topology_complete);
|
||||
assert_eq!(healthy.buckets.len(), 1);
|
||||
assert_eq!(healthy.set_buckets.len(), 1);
|
||||
let partial = store
|
||||
.list_buckets_for_admin_heal(Some(DiskSetSelector::new(1, 0)))
|
||||
.await
|
||||
.expect("partial scope");
|
||||
assert!(partial.topology_complete);
|
||||
assert_eq!(partial.buckets[0].name, "partial-bucket");
|
||||
let invalid = store
|
||||
.list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 1)))
|
||||
.await
|
||||
.expect("unconfigured scope");
|
||||
assert!(!invalid.topology_complete);
|
||||
assert!(invalid.buckets.is_empty());
|
||||
let offline = take_set_disks_offline(&store, &store.pools[1].disk_set[0], &[0]).await;
|
||||
assert!(store.list_buckets_for_admin_heal(None).await.is_err(), "quorum errors must remain errors");
|
||||
assert!(
|
||||
store
|
||||
.list_buckets_for_admin_heal(Some(DiskSetSelector::new(0, 0)))
|
||||
.await
|
||||
.expect("unrelated offline pool")
|
||||
.topology_complete
|
||||
);
|
||||
restore_set_disks(&store, &store.pools[1].disk_set[0], offline).await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_not_override_when_metadata_created_is_unix_epoch() {
|
||||
assert!(!should_override_created_from_metadata(OffsetDateTime::UNIX_EPOCH));
|
||||
|
||||
@@ -404,7 +404,17 @@ impl ECStore {
|
||||
expected: uuid::Uuid,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealResultItem> {
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
|
||||
self.run_bucket_heal_at_incarnation(bucket, expected, opts, move |store, bucket, mut opts| async move {
|
||||
if !opts.recreate && !opts.remove {
|
||||
let mut missing = false;
|
||||
for (pool, set) in store.heal_erasure_set_scopes(&opts).await? {
|
||||
missing |= store.pools[pool].get_disks(set).bucket_needs_reconstruction(&bucket).await?;
|
||||
}
|
||||
if missing {
|
||||
crate::bucket::metadata_sys::validate_bucket_reconstruction_in(&store.ctx, &bucket, expected).await?;
|
||||
opts.recreate = true;
|
||||
}
|
||||
}
|
||||
store.heal_bucket(&bucket, &opts).await
|
||||
})
|
||||
.await
|
||||
@@ -1451,6 +1461,178 @@ mod tests {
|
||||
.expect("minimal pool should build")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn admin_bucket_recovery_recreates_missing_directories_at_current_incarnation() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = "admin-bucket-recovery";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create a committed bucket generation");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("persisted generation");
|
||||
for disk in 1..4 {
|
||||
tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
|
||||
.await
|
||||
.expect("remove only an empty fixture bucket directory");
|
||||
}
|
||||
store
|
||||
.heal_bucket_at_incarnation(bucket, incarnation, &HealOpts::default())
|
||||
.await
|
||||
.expect("default administrator recovery should use committed generation authority");
|
||||
for disk in 0..4 {
|
||||
assert!(root.path().join(format!("pool0-disk{disk}/{bucket}")).is_dir());
|
||||
}
|
||||
assert_eq!(
|
||||
store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("recovered generation"),
|
||||
incarnation
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn admin_bucket_recovery_rejects_missing_corrupt_uncommitted_and_retired_authority() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
for state in ["missing", "corrupt", "uncommitted", "mismatched", "retired"] {
|
||||
let bucket = format!("recovery-authority-{state}");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create fixture generation");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(&bucket)
|
||||
.await
|
||||
.expect("fixture identity");
|
||||
let (mut metadata, persisted) = metadata_sys::get_config_from_disk_with_presence_in(&store.ctx, &bucket)
|
||||
.await
|
||||
.expect("committed metadata");
|
||||
assert!(persisted && metadata.bucket_creation_committed);
|
||||
let path = metadata.save_file_path();
|
||||
match state {
|
||||
"missing" => delete_config(store.clone(), &path).await.expect("remove metadata authority"),
|
||||
"corrupt" => save_config(store.clone(), &path, b"corrupt bucket metadata".to_vec())
|
||||
.await
|
||||
.expect("corrupt authority"),
|
||||
"uncommitted" | "mismatched" => {
|
||||
if state == "uncommitted" {
|
||||
metadata.bucket_creation_committed = false;
|
||||
} else {
|
||||
metadata.bucket_incarnation_id = Uuid::new_v4();
|
||||
}
|
||||
metadata
|
||||
.save_with_store_committed(store.clone())
|
||||
.await
|
||||
.expect("inject inconsistent metadata authority");
|
||||
}
|
||||
"retired" => {
|
||||
// A returning disk may retain a directory after retirement was committed.
|
||||
crate::bucket::retirement::commit_retirement(store.clone(), &bucket, incarnation, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("persist retirement evidence");
|
||||
}
|
||||
_ => unreachable!("fixed fixture states"),
|
||||
}
|
||||
for disk in 1..4 {
|
||||
tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
|
||||
.await
|
||||
.expect("lose fixture directories");
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.heal_bucket_at_incarnation(&bucket, incarnation, &HealOpts::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"must reject {state}"
|
||||
);
|
||||
for disk in 1..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0-disk{disk}/{bucket}")).exists(),
|
||||
"{state} cannot authorize reconstruction"
|
||||
);
|
||||
}
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn admin_bucket_recovery_rejects_offline_disks_and_fully_missing_sets() {
|
||||
for offline in [false, true] {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = "recovery-unproven-set";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket");
|
||||
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("fixture identity");
|
||||
let first_missing = usize::from(offline);
|
||||
for disk in first_missing..4 {
|
||||
tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
|
||||
.await
|
||||
.expect("remove fixture directories");
|
||||
}
|
||||
if offline {
|
||||
let disk = store.pools[0].disk_set[0].disks.read().await[0]
|
||||
.clone()
|
||||
.expect("online fixture disk");
|
||||
disk.close().await.expect("stop disk monitoring");
|
||||
disk.force_offline_for_test();
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.heal_bucket_at_incarnation(bucket, incarnation, &HealOpts::default())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for disk in first_missing..4 {
|
||||
assert!(!root.path().join(format!("pool0-disk{disk}/{bucket}")).exists());
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn admin_bucket_recovery_dry_run_preserves_missing_directories() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = "recovery-dry-run";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket");
|
||||
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("fixture identity");
|
||||
for disk in 1..4 {
|
||||
tokio::fs::remove_dir(root.path().join(format!("pool0-disk{disk}/{bucket}")))
|
||||
.await
|
||||
.expect("remove fixture directories");
|
||||
}
|
||||
store
|
||||
.heal_bucket_at_incarnation(
|
||||
bucket,
|
||||
incarnation,
|
||||
&HealOpts {
|
||||
dry_run: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("dry run can inspect eligible recovery");
|
||||
for disk in 1..4 {
|
||||
assert!(
|
||||
!root.path().join(format!("pool0-disk{disk}/{bucket}")).exists(),
|
||||
"dry run must not reconstruct directories"
|
||||
);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bucket_incarnation_heal_preserves_successor_shards_and_allows_fresh_repair() {
|
||||
|
||||
@@ -5030,6 +5030,339 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn repeated_abort_recovers_exact_orphan_before_decommission_drains() {
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use sha2::Digest;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create orphan recovery store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "orphan-multipart-drain", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-multipart-{}", uuid::Uuid::new_v4());
|
||||
let object = "exact-object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation");
|
||||
let lifecycle_guard = store.acquire_bucket_lifecycle_read_lock(&bucket).await.expect("bucket guard");
|
||||
let mut upload_opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &upload_opts)
|
||||
.await
|
||||
.expect("create source upload");
|
||||
drop(lifecycle_guard);
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial abort");
|
||||
|
||||
let upload_sha =
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
let staging_dir = |pool_index: usize, disk_index: usize| {
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool{pool_index}/set0/disk{disk_index}"))
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid)
|
||||
};
|
||||
for disk_index in 0..3 {
|
||||
let path = staging_dir(0, disk_index);
|
||||
tokio::fs::create_dir_all(&path).await.expect("create late part directory");
|
||||
tokio::fs::write(path.join("part.1"), b"late part after abort")
|
||||
.await
|
||||
.expect("write late part fixture");
|
||||
}
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
.await
|
||||
.expect("logical upload listing")
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
assert!(store.ensure_decommission_multipart_uploads_drained_for_test(0).await.is_err());
|
||||
assert!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, "different-key", &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"the same UploadID for a different key must not recover this path"
|
||||
);
|
||||
|
||||
// A minority metadata copy in another pool blocks recovery everywhere.
|
||||
let conflicting_metadata = staging_dir(1, 0).join(crate::disk::STORAGE_FORMAT_FILE);
|
||||
tokio::fs::create_dir_all(conflicting_metadata.parent().expect("metadata parent"))
|
||||
.await
|
||||
.expect("create conflicting pool staging");
|
||||
tokio::fs::write(&conflicting_metadata, b"unreadable metadata")
|
||||
.await
|
||||
.expect("write conflicting metadata");
|
||||
assert!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.is_err(),
|
||||
"no pool may be cleaned until every pool proves absence"
|
||||
);
|
||||
for disk_index in 0..3 {
|
||||
assert!(staging_dir(0, disk_index).join("part.1").exists());
|
||||
}
|
||||
tokio::fs::remove_file(conflicting_metadata)
|
||||
.await
|
||||
.expect("remove conflicting fixture");
|
||||
let (paused, release) = crate::store::multipart::pause_orphan_multipart_cleanup_for_test(&bucket);
|
||||
let recovery_store = store.clone();
|
||||
let recovery_bucket = bucket.clone();
|
||||
let recovery_upload = upload.upload_id.clone();
|
||||
let mut recovery = tokio::spawn(async move {
|
||||
recovery_store
|
||||
.abort_multipart_upload(&recovery_bucket, object, &recovery_upload, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
tokio::select! {
|
||||
paused = paused => paused.expect("cleanup owner announces arrival"),
|
||||
result = &mut recovery => panic!("recovery returned before physical cleanup: {result:?}"),
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("recovery should reach physical cleanup");
|
||||
recovery.abort();
|
||||
assert!(recovery.await.expect_err("request waiter is cancellable").is_cancelled());
|
||||
let upload_lock = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.new_ns_lock(crate::disk::RUSTFS_META_MULTIPART_BUCKET, &format!("{upload_sha}/{upload_uuid}"))
|
||||
.await
|
||||
.expect("same upload namespace");
|
||||
let mut upload_writer = Box::pin(upload_lock.get_write_lock(Duration::from_secs(10)));
|
||||
let mut bucket_writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
|
||||
assert!(
|
||||
futures::poll!(&mut upload_writer).is_pending(),
|
||||
"cancelled recovery must retain the upload lock"
|
||||
);
|
||||
assert!(
|
||||
futures::poll!(&mut bucket_writer).is_pending(),
|
||||
"cancelled recovery must retain the real bucket guard"
|
||||
);
|
||||
release.send(()).expect("allow physical cleanup to finish");
|
||||
let (upload_guard, bucket_guard) =
|
||||
tokio::time::timeout(Duration::from_secs(10), async { tokio::join!(upload_writer, bucket_writer) })
|
||||
.await
|
||||
.expect("both guards should release after physical cleanup");
|
||||
drop(upload_guard.expect("upload writer resumes"));
|
||||
drop(bucket_guard.expect("bucket writer resumes"));
|
||||
for pool_index in 0..2 {
|
||||
for disk_index in 0..4 {
|
||||
assert!(!staging_dir(pool_index, disk_index).exists(), "every checked staging path must be absent");
|
||||
}
|
||||
}
|
||||
store
|
||||
.ensure_decommission_multipart_uploads_drained_for_test(0)
|
||||
.await
|
||||
.expect("physical decommission gate should open after recovery");
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::InvalidUploadID(_, _, _))
|
||||
),
|
||||
"an absent upload must not claim to have recovered anything"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn repeated_abort_keeps_no_such_upload_while_a_disk_is_offline() {
|
||||
use sha2::Digest;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create offline-disk recovery store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "orphan-multipart-offline", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-multipart-{}", uuid::Uuid::new_v4());
|
||||
let object = "offline-object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("bucket incarnation");
|
||||
let lifecycle_guard = store.acquire_bucket_lifecycle_read_lock(&bucket).await.expect("bucket guard");
|
||||
let mut upload_opts = ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
};
|
||||
upload_opts.add_bucket_lifecycle_lock_guard(&lifecycle_guard);
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &upload_opts)
|
||||
.await
|
||||
.expect("create source upload");
|
||||
drop(lifecycle_guard);
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("initial abort");
|
||||
|
||||
let upload_sha =
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
let staging_dir = |disk_index: usize| {
|
||||
temp_dir
|
||||
.path()
|
||||
.join(format!("pool0/set0/disk{disk_index}"))
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&upload_sha)
|
||||
.join(&upload_uuid)
|
||||
};
|
||||
for disk_index in 0..3 {
|
||||
let path = staging_dir(disk_index);
|
||||
tokio::fs::create_dir_all(&path).await.expect("create late part directory");
|
||||
tokio::fs::write(path.join("part.1"), b"late part after abort")
|
||||
.await
|
||||
.expect("write late part fixture");
|
||||
}
|
||||
|
||||
// One offline disk means absence cannot be proven. The idempotent Abort
|
||||
// must keep answering NoSuchUpload instead of surfacing a disk error,
|
||||
// and must not delete anything.
|
||||
let set = store.pools[0].get_disks_by_key(object);
|
||||
let offline = set.disks.write().await[3].take();
|
||||
assert!(offline.is_some(), "fixture disk must have been online");
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::InvalidUploadID(_, _, _))
|
||||
),
|
||||
"unprovable absence must keep NoSuchUpload while a disk is offline"
|
||||
);
|
||||
for disk_index in 0..3 {
|
||||
assert!(
|
||||
staging_dir(disk_index).join("part.1").exists(),
|
||||
"no staging may be deleted while a disk is offline"
|
||||
);
|
||||
}
|
||||
|
||||
set.disks.write().await[3] = offline;
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recovery resumes once every disk is observable");
|
||||
for disk_index in 0..4 {
|
||||
assert!(!staging_dir(disk_index).exists(), "recovered staging must be absent");
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn orphan_multipart_recovery_preserves_current_bucket_incarnation() {
|
||||
use sha2::Digest;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create bucket lifetime recovery store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "orphan-bucket-lifetime", &[4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("orphan-lifetime-{}", uuid::Uuid::new_v4());
|
||||
let object = "object.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create original bucket");
|
||||
let original_incarnation = store.bucket_incarnation_id(&bucket).await.expect("original incarnation");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create original upload");
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("abort original upload");
|
||||
store
|
||||
.delete_bucket(&bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete original bucket");
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
assert_ne!(store.bucket_incarnation_id(&bucket).await.expect("new incarnation"), original_incarnation);
|
||||
let mut reader = PutObjReader::from_vec(b"current bucket object".to_vec());
|
||||
store
|
||||
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write current bucket object");
|
||||
let current_upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create current incarnation upload for the same key");
|
||||
|
||||
let upload_sha =
|
||||
hex_simd::encode_to_string(sha2::Sha256::digest(format!("{bucket}/{object}").as_bytes()), hex_simd::AsciiCase::Lower);
|
||||
let upload_uuid = crate::runtime::sources::upload_uuid_suffix(&upload.upload_id);
|
||||
let leftover = temp_dir
|
||||
.path()
|
||||
.join("pool0/set0/disk0")
|
||||
.join(crate::disk::RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(upload_sha)
|
||||
.join(upload_uuid)
|
||||
.join("part.1");
|
||||
tokio::fs::create_dir_all(leftover.parent().expect("staging parent"))
|
||||
.await
|
||||
.expect("create old incarnation staging");
|
||||
tokio::fs::write(&leftover, b"previous bucket lifetime")
|
||||
.await
|
||||
.expect("write previous incarnation fixture");
|
||||
for opts in [
|
||||
ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(original_incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(uuid::Uuid::nil()),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert!(
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &opts)
|
||||
.await
|
||||
.is_err(),
|
||||
"a stale or nil current-bucket identity must fail closed"
|
||||
);
|
||||
assert!(leftover.exists(), "failed bucket lifetime proof must preserve the path");
|
||||
}
|
||||
store
|
||||
.abort_multipart_upload(&bucket, object, &upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("exact metadata-less staging from an old incarnation is unreferenced");
|
||||
assert!(!leftover.exists());
|
||||
store
|
||||
.get_multipart_info(&bucket, object, ¤t_upload.upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("recovery must preserve the current incarnation's active upload");
|
||||
let mut reader = store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("current object must remain readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("read current object");
|
||||
assert_eq!(body, b"current bucket object");
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn active_multipart_upload_routes_before_faulted_suspended_source() {
|
||||
@@ -18591,6 +18924,70 @@ mod tests {
|
||||
assert_eq!(body, b"new-generation");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial_test::serial]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn cancelled_ecstore_put_part_keeps_bucket_lifecycle_lock_through_commit() {
|
||||
for pause in [
|
||||
crate::set_disk::MultipartCommitPause::PutPartAfterCapacityAdmission,
|
||||
crate::set_disk::MultipartCommitPause::PutPartAfterRename,
|
||||
] {
|
||||
let temp_dir = tempfile::tempdir().expect("create cancelled multipart lifecycle store dir");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "cancelled-multipart-lifecycle", &[4, 4]))
|
||||
.await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = format!("cancelled-part-lifecycle-{}", Uuid::new_v4());
|
||||
let object = "part.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for multipart lifecycle cancellation");
|
||||
let upload = store
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload through the owning ECStore");
|
||||
let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause);
|
||||
let put_store = Arc::clone(&store);
|
||||
let put_bucket = bucket.clone();
|
||||
let put_upload_id = upload.upload_id.clone();
|
||||
let put = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![0x45; 4096]);
|
||||
put_store
|
||||
.put_object_part(&put_bucket, object, &put_upload_id, 1, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
put.abort();
|
||||
assert!(
|
||||
put.await
|
||||
.expect_err("the ECStore request waiter must remain cancellable")
|
||||
.is_cancelled()
|
||||
);
|
||||
|
||||
let mut writer = Box::pin(store.acquire_bucket_lifecycle_write_lock(&bucket));
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(100), &mut writer).await.is_err(),
|
||||
"canceling UploadPart at {pause:?} must not release its bucket lifecycle owner before the commit drains"
|
||||
);
|
||||
barrier.release();
|
||||
let lifecycle_guard = tokio::time::timeout(Duration::from_secs(10), writer)
|
||||
.await
|
||||
.expect("the bucket lifecycle writer must resume after the part commit drains")
|
||||
.expect("acquire the bucket lifecycle write lock after cancellation");
|
||||
drop(lifecycle_guard);
|
||||
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the cancelled request's owned part commit must finish successfully");
|
||||
assert_eq!(parts.parts.len(), 1, "the commit must publish exactly the uploaded part at {pause:?}");
|
||||
assert_eq!(parts.parts[0].part_num, 1);
|
||||
assert_eq!(parts.parts[0].size, 4096);
|
||||
shutdown.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn pre_upgrade_multipart_upload_can_complete_and_abort_in_the_same_bucket_lifetime() {
|
||||
@@ -22143,7 +22540,6 @@ mod tests {
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("kill-point bucket should be created");
|
||||
let _compaction_proof = install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
|
||||
for (index, point, expected_state, expected_revision, expected_committed, expected_recovered) in [
|
||||
(
|
||||
@@ -22187,10 +22583,14 @@ mod tests {
|
||||
.await
|
||||
.expect("kill-point source should be written");
|
||||
let remote_before = backend.object_count().await;
|
||||
// Earlier cancellation/recovery cases must not consume this
|
||||
// transaction's 30-second homogeneous-fleet proof window.
|
||||
let _compaction_proof =
|
||||
install_transition_transaction_compaction_fleet_proof_for_test("object-transaction-fencing-test");
|
||||
let barrier = TransitionTransactionKillPointBarrier::install(bucket, &object, point);
|
||||
let transition_store = store.clone();
|
||||
let transition_object = object.clone();
|
||||
let transition = tokio::spawn(async move {
|
||||
let mut transition = tokio::spawn(async move {
|
||||
transition_store
|
||||
.transition_object(
|
||||
bucket,
|
||||
@@ -22209,7 +22609,12 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::select! {
|
||||
_ = barrier.wait_until_paused() => {}
|
||||
result = &mut transition => {
|
||||
panic!("transition exited before reaching {point:?}: {result:?}");
|
||||
}
|
||||
}
|
||||
transition.abort();
|
||||
assert!(
|
||||
transition
|
||||
|
||||
@@ -32,6 +32,42 @@ fn data_movement_multipart_discovery_counts() -> &'static std::sync::Mutex<std::
|
||||
DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type OrphanMultipartCleanupPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>);
|
||||
|
||||
#[cfg(test)]
|
||||
static ORPHAN_MULTIPART_CLEANUP_PAUSES: std::sync::OnceLock<
|
||||
std::sync::Mutex<std::collections::HashMap<String, OrphanMultipartCleanupPause>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn pause_orphan_multipart_cleanup_for_test(
|
||||
bucket: &str,
|
||||
) -> (tokio::sync::oneshot::Receiver<()>, tokio::sync::oneshot::Sender<()>) {
|
||||
let (arrived_tx, arrived_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let previous = ORPHAN_MULTIPART_CLEANUP_PAUSES
|
||||
.get_or_init(Default::default)
|
||||
.lock()
|
||||
.expect("orphan cleanup pause mutex should not poison")
|
||||
.insert(bucket.to_owned(), (arrived_tx, release_rx));
|
||||
assert!(previous.is_none(), "each bucket has at most one orphan cleanup pause");
|
||||
(arrived_rx, release_tx)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_orphan_multipart_cleanup(bucket: &str) {
|
||||
let pause = ORPHAN_MULTIPART_CLEANUP_PAUSES
|
||||
.get_or_init(Default::default)
|
||||
.lock()
|
||||
.expect("orphan cleanup pause mutex should not poison")
|
||||
.remove(bucket);
|
||||
if let Some((arrived, release)) = pause {
|
||||
let _ = arrived.send(());
|
||||
let _ = release.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn decommission_multipart_target_clear_pending(opts: &ObjectOptions, target: Option<&ObjectInfo>) -> Result<bool> {
|
||||
let expected_mod_time = opts.mod_time.ok_or_else(|| Error::DecommissionCapacityBlocked {
|
||||
message: "multipart cleanup cannot prove exact target absence without a modification time".to_string(),
|
||||
@@ -654,7 +690,10 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
if let Some(guard) = bucket_lifecycle_guard {
|
||||
opts.bucket_lifecycle_lock_owner = Some(Arc::new(guard));
|
||||
}
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
@@ -750,19 +789,12 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let opts = &opts;
|
||||
|
||||
// TODO(backlog): defer DeleteUploadID to background for faster abort response
|
||||
|
||||
if self.single_pool() {
|
||||
return self.pools[0].abort_multipart_upload(bucket, object, upload_id, opts).await;
|
||||
}
|
||||
let (opts, bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
let pool = &self.pools[pool_idx];
|
||||
|
||||
let err = match pool.abort_multipart_upload(bucket, object, upload_id, opts).await {
|
||||
let err = match pool.abort_multipart_upload(bucket, object, upload_id, &opts).await {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(err) => {
|
||||
//
|
||||
@@ -775,7 +807,92 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||
self.recover_orphan_multipart_upload(bucket, object, upload_id, opts, bucket_lifecycle_guard)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn recover_orphan_multipart_upload(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
opts: ObjectOptions,
|
||||
bucket_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<()> {
|
||||
let invalid_upload = || StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned());
|
||||
if self.ctx.lock_manager().is_disabled() {
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
let Some(bucket_lifecycle_guard) = bucket_lifecycle_guard else {
|
||||
// A signal borrowed from an enclosing operation cannot keep its lock
|
||||
// alive when the request waiter disappears during physical cleanup.
|
||||
return Err(invalid_upload());
|
||||
};
|
||||
let Some(expected_incarnation) = opts.expected_bucket_incarnation_id.filter(|id| !id.is_nil()) else {
|
||||
return Err(invalid_upload());
|
||||
};
|
||||
// Recovery is a side path of an idempotent Abort. Until the first
|
||||
// deletion is issued, anything that stops staging absence from being
|
||||
// proven (an offline disk, lock contention, partially overlapping lock
|
||||
// domains, unreadable bucket metadata) keeps the upload reported as
|
||||
// gone, exactly as before recovery existed; only failures inside the
|
||||
// deletion task are request failures, because a retry can finish them.
|
||||
let (metadata, persisted) =
|
||||
match crate::bucket::metadata_sys::get_config_from_disk_with_presence_in(&self.ctx, bucket).await {
|
||||
Ok(loaded) => loaded,
|
||||
Err(error) => {
|
||||
debug!(bucket, object, upload_id, %error, "orphan multipart cleanup skipped: bucket metadata unavailable");
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
};
|
||||
if !persisted
|
||||
|| !metadata.bucket_creation_committed
|
||||
|| !metadata.bucket_incarnation_sidecar
|
||||
|| metadata.bucket_incarnation_id != expected_incarnation
|
||||
{
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
|
||||
// Lock order: bucket lifecycle, then upload locks in pool-index order.
|
||||
// Prove metadata absence across every pool before deleting any staging.
|
||||
let mut cleanup_guards = Vec::with_capacity(self.pools.len());
|
||||
for pool in &self.pools {
|
||||
let guard = match pool
|
||||
.get_disks_by_key(object)
|
||||
.lock_orphan_multipart_cleanup(bucket, object, upload_id, &opts, &cleanup_guards)
|
||||
.await
|
||||
{
|
||||
Ok(guard) => guard,
|
||||
Err(error) => {
|
||||
if !is_err_invalid_upload_id(&error) {
|
||||
debug!(bucket, object, upload_id, %error, "orphan multipart cleanup skipped: staging absence not proven");
|
||||
}
|
||||
return Err(invalid_upload());
|
||||
}
|
||||
};
|
||||
cleanup_guards.push(guard);
|
||||
}
|
||||
let bucket = bucket.to_owned();
|
||||
let object = object.to_owned();
|
||||
let upload_id = upload_id.to_owned();
|
||||
tokio::spawn(async move {
|
||||
// Own both the real bucket guard and all upload guards until every
|
||||
// disk deletion settles, including after cancellation of the waiter.
|
||||
let _bucket_lifecycle_guard = bucket_lifecycle_guard;
|
||||
#[cfg(test)]
|
||||
pause_orphan_multipart_cleanup(&bucket).await;
|
||||
let mut recovered = false;
|
||||
for guard in &cleanup_guards {
|
||||
recovered |= guard.delete(&bucket, &object, &opts).await?;
|
||||
}
|
||||
if recovered {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(StorageError::InvalidUploadID(bucket, object, upload_id))
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|error| Error::other_with_context("orphan multipart cleanup task failed", error))?
|
||||
}
|
||||
|
||||
pub(crate) async fn abort_multipart_upload_for_data_movement(
|
||||
|
||||
@@ -18,6 +18,11 @@ use crate::disk::error::{DiskError, Result as DiskResult};
|
||||
use crate::disk::{DeleteOptions, Disk, RenameDataGuards, RenameDataResp};
|
||||
use crate::runtime::instance::{InstanceContext, NamespaceCommitGuard};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use rustfs_io_metrics::{
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD,
|
||||
PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL,
|
||||
};
|
||||
use tracing::{debug, error};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
@@ -331,15 +336,38 @@ async fn rename_local_data_with_ctx(
|
||||
destination: (&str, &str),
|
||||
mut guards: RenameDataGuards,
|
||||
) -> DiskResult<RenameDataResp> {
|
||||
let handler_started = rustfs_io_metrics::put_stage_timer();
|
||||
let lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let lookup_ms = lookup_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
|
||||
if let Some(duration_ms) = lookup_ms {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOOKUP, duration_ms);
|
||||
}
|
||||
let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0);
|
||||
let admit_started = rustfs_io_metrics::put_stage_timer();
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?;
|
||||
let admit_ms = admit_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
|
||||
if let Some(duration_ms) = admit_ms {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_ADMIT, duration_ms);
|
||||
}
|
||||
guards.namespace_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let local_rename_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = disk
|
||||
.rename_data_borrowed_with_fence_observed(source.0, source.1, fi, destination.0, destination.1, guards)
|
||||
.await
|
||||
.result;
|
||||
let local_rename_ms = local_rename_started.map(|started| started.elapsed().as_secs_f64() * 1000.0);
|
||||
if let Some(duration_ms) = local_rename_ms {
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_LOCAL_RENAME, duration_ms);
|
||||
}
|
||||
drop(owner);
|
||||
if let Some(handler_started) = handler_started {
|
||||
let handler_ms = handler_started.elapsed().as_secs_f64() * 1000.0;
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_TOTAL, handler_ms);
|
||||
let child_ms = lookup_ms.unwrap_or(0.0) + admit_ms.unwrap_or(0.0) + local_rename_ms.unwrap_or(0.0);
|
||||
let overhead_ms = (handler_ms - child_ms).max(0.0);
|
||||
rustfs_io_metrics::record_put_object_stage_duration(PUT_STAGE_SET_DISK_RENAME_REMOTE_HANDLER_OVERHEAD, overhead_ms);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
@@ -1669,7 +1697,9 @@ mod tests {
|
||||
#[serial_test::serial]
|
||||
async fn target_ordinary_timeout_keeps_its_physical_namespace_owner() {
|
||||
use crate::disk::os::prepared_publication_test_hooks as hooks;
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("1"))], async {
|
||||
// Allow real fixture I/O to finish under load. Expire the operation
|
||||
// with virtual time only after its physical publication is paused.
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, Some("60"))], async {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let store = super::super::tests::build_store_with_ctx(ctx.clone());
|
||||
let root = tempfile::tempdir().expect("root");
|
||||
@@ -1703,7 +1733,7 @@ mod tests {
|
||||
.await
|
||||
.expect("bounded entry");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(std::time::Duration::from_secs(2)).await;
|
||||
tokio::time::advance(std::time::Duration::from_secs(61)).await;
|
||||
tokio::time::resume();
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut rename)
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![cfg(target_os = "linux")]
|
||||
|
||||
use bytes::Bytes;
|
||||
mod storage_api;
|
||||
|
||||
use std::time::Duration;
|
||||
use storage_api::cleanup_isolation::{DeleteOptions, DiskAPI, DiskError, DiskOption, Endpoint, new_disk};
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground() {
|
||||
const CHILD: &str = "RUSTFS_CLEANUP_LINUX_TEST_CHILD";
|
||||
let foreground = rustix::thread::sched_getaffinity(None).unwrap();
|
||||
let cpu = (0..rustix::thread::CpuSet::MAX_CPU)
|
||||
.find(|cpu| foreground.is_set(*cpu))
|
||||
.expect("the test process must have an allowed CPU");
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
for mode in ["custom", "default", "none", "restricted"] {
|
||||
if mode == "default" && (!foreground.is_set(1) || !foreground.is_set(2)) {
|
||||
continue;
|
||||
}
|
||||
let output = tokio::task::spawn_blocking(move || {
|
||||
let mut command = std::process::Command::new(std::env::current_exe().unwrap());
|
||||
command
|
||||
.args([
|
||||
"--exact",
|
||||
"cleanup_disk_and_periodic_gc_use_affinity_without_changing_foreground",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD, mode)
|
||||
.env("RUSTFS_CLEANUP_ISOLATE_ENABLE", "true")
|
||||
.env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_MAX_PENDING", "4")
|
||||
.env("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_DISK_MAX_PENDING", "4")
|
||||
.env("RUSTFS_CLEANUP_DISK_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_GC_WORKERS", "1")
|
||||
.env("RUSTFS_CLEANUP_BLOCKING_THREADS", "1")
|
||||
.env("RUSTFS_CLEANUP_ASYNC_THREADS", "1")
|
||||
.env_remove("RUSTFS_CLEANUP_CPUS")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_DEFER_HOLD_WORKER")
|
||||
.env_remove("RUSTFS_PUT_RENAME_TAIL_CLEANUP_ZERO_TARGET_TMP_DELETE_SKIP");
|
||||
match mode {
|
||||
"custom" => {
|
||||
command.env("RUSTFS_CLEANUP_CPUS", cpu.to_string());
|
||||
}
|
||||
"none" => {
|
||||
command.env("RUSTFS_CLEANUP_CPUS", "none");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
command.output().unwrap()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let mode = std::env::var(CHILD).unwrap();
|
||||
if mode == "restricted" {
|
||||
let mut restricted = rustix::thread::CpuSet::new();
|
||||
restricted.set(cpu);
|
||||
rustix::thread::sched_setaffinity(None, &restricted).expect("restrict initialization to one allowed CPU");
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let disk = new_disk(&Endpoint::try_from(dir.path().to_str().unwrap()).unwrap(), &DiskOption::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let bucket = "cleanup-affinity";
|
||||
disk.make_volume(bucket).await.unwrap();
|
||||
disk.write_all(bucket, "old/part.1", Bytes::from_static(b"garbage"))
|
||||
.await
|
||||
.unwrap();
|
||||
let receipt = disk
|
||||
.write_all(bucket, "old/.rustfs-old-data-cleanup-receipt.json", Bytes::from_static(b"receipt"))
|
||||
.await;
|
||||
if mode == "restricted" {
|
||||
assert!(
|
||||
receipt
|
||||
.expect_err("unavailable default CPUs must reject cleanup")
|
||||
.to_string()
|
||||
.contains("allowed CPU set")
|
||||
);
|
||||
assert_eq!(disk.read_all(bucket, "old/part.1").await.unwrap(), Bytes::from_static(b"garbage"));
|
||||
return;
|
||||
}
|
||||
receipt.unwrap();
|
||||
let expected_mask = match mode.as_str() {
|
||||
"default" => "1-2".to_owned(),
|
||||
"none" => std::fs::read_to_string("/proc/thread-self/status")
|
||||
.unwrap()
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("Cpus_allowed_list:"))
|
||||
.unwrap()
|
||||
.trim()
|
||||
.to_owned(),
|
||||
_ => cpu.to_string(),
|
||||
};
|
||||
disk.write_all(bucket, "live/part.1", Bytes::from_static(b"keep"))
|
||||
.await
|
||||
.unwrap();
|
||||
disk.delete(
|
||||
bucket,
|
||||
"old",
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(matches!(disk.read_all(bucket, "old/part.1").await, Err(DiskError::FileNotFound)));
|
||||
|
||||
// One async worker and one lazily started blocking worker. Inspect the
|
||||
// real process threads, including the threads executing filesystem work.
|
||||
let mut cleanup_threads = 0;
|
||||
for entry in std::fs::read_dir("/proc/self/task").unwrap() {
|
||||
let task = entry.unwrap().path();
|
||||
let Ok(name) = std::fs::read_to_string(task.join("comm")) else {
|
||||
continue;
|
||||
};
|
||||
if name.trim() != "rustfs-cleanup" {
|
||||
continue;
|
||||
}
|
||||
cleanup_threads += 1;
|
||||
let status = std::fs::read_to_string(task.join("status")).unwrap();
|
||||
let mask = status
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("Cpus_allowed_list:"))
|
||||
.unwrap()
|
||||
.trim();
|
||||
assert_eq!(mask, expected_mask);
|
||||
}
|
||||
assert_eq!(cleanup_threads, 2, "async and blocking workers must both use the configured runtime");
|
||||
assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground);
|
||||
|
||||
let trash = dir.path().join(".rustfs.sys/tmp/.trash");
|
||||
assert!(std::fs::read_dir(&trash).unwrap().next().is_some());
|
||||
// Advance the foreground timer that dispatches the five-minute periodic
|
||||
// scan; the dedicated runtime then performs the actual physical removal.
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(301)).await;
|
||||
tokio::time::resume();
|
||||
tokio::time::timeout(Duration::from_secs(20), async {
|
||||
while std::fs::read_dir(&trash).unwrap().next().is_some() {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("periodic GC must physically remove trash");
|
||||
assert_eq!(disk.read_all(bucket, "live/part.1").await.unwrap(), Bytes::from_static(b"keep"));
|
||||
assert_eq!(rustix::thread::sched_getaffinity(None).unwrap(), foreground);
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![cfg(all(target_os = "linux", feature = "test-util"))]
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use bytes::Bytes;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo};
|
||||
use std::{path::Path, time::Duration};
|
||||
use storage_api::fd_relative::{
|
||||
DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, LocalPublicationPause, LocalPublicationStage, ReadOptions, new_disk,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
const RUSTFS_META_TMP_BUCKET: &str = ".rustfs.sys/tmp";
|
||||
|
||||
async fn disk_at(path: &Path) -> DiskStore {
|
||||
new_disk(
|
||||
&Endpoint::try_from(path.to_str().expect("UTF-8 test path")).expect("endpoint"),
|
||||
&DiskOption::default(),
|
||||
)
|
||||
.await
|
||||
.expect("local disk")
|
||||
}
|
||||
|
||||
fn metadata(data: Bytes, inline: bool, directory: Uuid) -> FileInfo {
|
||||
let mut fi = FileInfo::new("object", 1, 0);
|
||||
fi.erasure.index = 1;
|
||||
fi.version_id = Some(Uuid::nil());
|
||||
fi.data_dir = Some(directory);
|
||||
fi.size = i64::try_from(data.len()).expect("test data size");
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: data.len(),
|
||||
actual_size: fi.size,
|
||||
..Default::default()
|
||||
}];
|
||||
fi.mod_time = Some(OffsetDateTime::now_utc());
|
||||
fi.data = inline.then_some(data);
|
||||
if inline {
|
||||
fi.set_inline_data();
|
||||
}
|
||||
fi
|
||||
}
|
||||
|
||||
async fn put(disk: &DiskStore, staging: &str, data: Bytes, inline: bool) -> Uuid {
|
||||
let directory = Uuid::new_v4();
|
||||
if !inline {
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("{staging}/{directory}/part.1"), data.clone())
|
||||
.await
|
||||
.expect("stage shard");
|
||||
}
|
||||
disk.rename_data(RUSTFS_META_TMP_BUCKET, staging, metadata(data, inline, directory), "bucket", "object")
|
||||
.await
|
||||
.expect("commit object");
|
||||
directory
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pinned_root_reads_and_writes_original_disk_after_path_replacement() {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let root = temp.path().join("disk");
|
||||
std::fs::create_dir(&root).expect("disk root");
|
||||
let disk = disk_at(&root).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
disk.write_all("bucket", "metadata", Bytes::from_static(b"original"))
|
||||
.await
|
||||
.expect("initial write");
|
||||
let moved = temp.path().join("original");
|
||||
std::fs::rename(&root, &moved).expect("move mount path");
|
||||
std::fs::create_dir_all(root.join("bucket")).expect("replacement disk");
|
||||
std::fs::write(root.join("bucket/metadata"), b"replacement").expect("replacement sentinel");
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", "metadata").await.expect("pinned read"),
|
||||
Bytes::from_static(b"original")
|
||||
);
|
||||
disk.write_all("bucket", "metadata", Bytes::from_static(b"updated"))
|
||||
.await
|
||||
.expect("pinned write");
|
||||
assert_eq!(std::fs::read(moved.join("bucket/metadata")).expect("original disk"), b"updated");
|
||||
assert_eq!(std::fs::read(root.join("bucket/metadata")).expect("replacement disk"), b"replacement");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn relative_reads_and_writes_reject_symlink_escape() {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
let outside = tempfile::tempdir().expect("outside directory");
|
||||
std::fs::write(outside.path().join("sentinel"), b"preserved").expect("sentinel");
|
||||
std::os::unix::fs::symlink(outside.path(), temp.path().join("bucket/link")).expect("directory link");
|
||||
std::os::unix::fs::symlink(outside.path().join("sentinel"), temp.path().join("bucket/file-link")).expect("file link");
|
||||
for path in ["link/sentinel", "file-link"] {
|
||||
assert!(matches!(disk.read_all("bucket", path).await, Err(DiskError::InvalidPath)));
|
||||
assert!(matches!(
|
||||
disk.write_all("bucket", path, Bytes::from_static(b"unsafe")).await,
|
||||
Err(DiskError::InvalidPath)
|
||||
));
|
||||
}
|
||||
assert_eq!(std::fs::read(outside.path().join("sentinel")).expect("outside bytes"), b"preserved");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_and_non_inline_overwrites_remain_readable_after_reopen() {
|
||||
for inline in [true, false] {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
put(&disk, "first", Bytes::from_static(b"old bytes"), inline).await;
|
||||
let directory = put(&disk, "second", Bytes::from_static(b"new bytes"), inline).await;
|
||||
drop(disk);
|
||||
let disk = disk_at(temp.path()).await;
|
||||
let fi = disk
|
||||
.read_version(
|
||||
"bucket",
|
||||
"bucket",
|
||||
"object",
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("committed metadata after reopen");
|
||||
assert_eq!(fi.data_dir, Some(directory));
|
||||
if inline {
|
||||
assert_eq!(fi.data, Some(Bytes::from_static(b"new bytes")));
|
||||
} else {
|
||||
assert_eq!(
|
||||
disk.read_all("bucket", &format!("object/{directory}/part.1"))
|
||||
.await
|
||||
.expect("shard bytes"),
|
||||
Bytes::from_static(b"new bytes")
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rename_rejects_symlinked_destination_metadata() {
|
||||
for inline in [true, false] {
|
||||
let temp = tempfile::tempdir().expect("disk directory");
|
||||
let outside = tempfile::tempdir().expect("outside directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
put(&disk, "seed", Bytes::from_static(b"old object"), inline).await;
|
||||
let destination = temp.path().join("bucket/object/xl.meta");
|
||||
let original = std::fs::read(&destination).expect("valid existing metadata");
|
||||
let external_metadata = outside.path().join("xl.meta");
|
||||
std::fs::write(&external_metadata, &original).expect("external valid metadata");
|
||||
std::fs::remove_file(&destination).expect("replace destination metadata");
|
||||
std::os::unix::fs::symlink(&external_metadata, &destination).expect("metadata escape link");
|
||||
|
||||
let directory = Uuid::new_v4();
|
||||
let data = Bytes::from_static(b"replacement");
|
||||
if !inline {
|
||||
disk.write_all(RUSTFS_META_TMP_BUCKET, &format!("replacement/{directory}/part.1"), data.clone())
|
||||
.await
|
||||
.expect("stage non-inline data");
|
||||
}
|
||||
let result = disk
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"replacement",
|
||||
metadata(data, inline, directory),
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(DiskError::InvalidPath)),
|
||||
"symlinked metadata must be rejected: {result:?}"
|
||||
);
|
||||
assert_eq!(std::fs::read_link(&destination).expect("link must remain"), external_metadata);
|
||||
assert_eq!(std::fs::read(&external_metadata).expect("external metadata must survive"), original);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn prepared_publication_rejects_source_or_directory_replacement() {
|
||||
for replace_directory in [false, true] {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
put(&disk, "first", Bytes::from_static(b"old bytes"), true).await;
|
||||
let old = std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata");
|
||||
let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename)
|
||||
.expect("publication pause");
|
||||
let writer = disk.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
writer
|
||||
.rename_data(
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
"second",
|
||||
metadata(Bytes::from_static(b"new bytes"), true, Uuid::new_v4()),
|
||||
"bucket",
|
||||
"object",
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), pause.entered())
|
||||
.await
|
||||
.expect("publication reached pause")
|
||||
.expect("pause entered");
|
||||
if replace_directory {
|
||||
std::fs::rename(temp.path().join("bucket/object"), temp.path().join("detached")).expect("detach object");
|
||||
std::fs::create_dir(temp.path().join("bucket/object")).expect("replacement directory");
|
||||
std::fs::write(temp.path().join("bucket/object/xl.meta"), &old).expect("replacement metadata");
|
||||
} else {
|
||||
let src = temp.path().join(RUSTFS_META_TMP_BUCKET).join("second/xl.meta");
|
||||
std::fs::rename(&src, src.with_extension("detached")).expect("detach staged metadata");
|
||||
std::fs::write(src, b"unrelated").expect("replacement staged file");
|
||||
}
|
||||
drop(pause);
|
||||
assert!(task.await.expect("writer task").is_err());
|
||||
assert_eq!(
|
||||
std::fs::read(temp.path().join("bucket/object/xl.meta")).expect("old metadata survives"),
|
||||
old
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn cancelled_waiter_keeps_publication_serialized_until_physical_completion() {
|
||||
let temp = tempfile::tempdir().expect("test directory");
|
||||
let disk = disk_at(temp.path()).await;
|
||||
disk.make_volume("bucket").await.expect("bucket");
|
||||
put(&disk, "first", Bytes::from_static(b"old bytes"), true).await;
|
||||
let mut pause = LocalPublicationPause::install(&disk, "bucket", "object/xl.meta", LocalPublicationStage::PreparedRename)
|
||||
.expect("publication pause");
|
||||
let writer = disk.clone();
|
||||
let pending = tokio::spawn(async move { put(&writer, "shared-stage", Bytes::from_static(b"cancelled waiter"), true).await });
|
||||
tokio::time::timeout(Duration::from_secs(30), pause.entered())
|
||||
.await
|
||||
.expect("publication reached pause")
|
||||
.expect("pause entered");
|
||||
pending.abort();
|
||||
assert!(pending.await.expect_err("waiter aborted").is_cancelled());
|
||||
let writer = disk.clone();
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let mut retry = tokio::spawn(async move {
|
||||
let _ = started_tx.send(());
|
||||
put(&writer, "shared-stage", Bytes::from_static(b"latest bytes"), true).await
|
||||
});
|
||||
started_rx.await.expect("retry started");
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_millis(200), &mut retry).await.is_err(),
|
||||
"a retry must not publish while the cancelled writer still owns physical IO"
|
||||
);
|
||||
drop(pause);
|
||||
tokio::time::timeout(Duration::from_secs(30), retry)
|
||||
.await
|
||||
.expect("retry completed")
|
||||
.expect("retry task");
|
||||
let fi = disk
|
||||
.read_version(
|
||||
"bucket",
|
||||
"bucket",
|
||||
"object",
|
||||
"",
|
||||
&ReadOptions {
|
||||
read_data: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("latest metadata");
|
||||
assert_eq!(fi.data, Some(Bytes::from_static(b"latest bytes")));
|
||||
}
|
||||
@@ -18,6 +18,7 @@ mod storage_api;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use storage_api::contract_compat::ECStore;
|
||||
use storage_api::metadata_lock::{
|
||||
BucketOperations, CompletePart, Error, MakeBucketOptions, MultipartOperations, NamespaceLocking, ObjectIO, ObjectOperations,
|
||||
ObjectOptions, PutObjReader, PutObjectCommitBarrier, PutObjectCommitPause, init_bucket_metadata_sys,
|
||||
@@ -170,6 +171,100 @@ async fn reused_metadata_lock_still_enforces_quota() {
|
||||
.expect("acquire writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_quota_read_does_not_ignore_later_quota() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "authoritative-quota-quota-set";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
|
||||
let mut first = PutObjReader::from_vec(b"cache no quota".to_vec());
|
||||
store
|
||||
.put_object(bucket, "warm", &mut first, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("warm no-quota metadata cache");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("set quota after no-quota cache was observed");
|
||||
|
||||
let mut opts = quota_snapshot_options(&store, bucket).await;
|
||||
assert!(opts.set_quota_admission(0, 1));
|
||||
let mut data = PutObjReader::from_vec(b"too large".to_vec());
|
||||
let error = store
|
||||
.put_object(bucket, "object", &mut data, &opts)
|
||||
.await
|
||||
.expect_err("quota set after a no-quota cache hit must still reject growth");
|
||||
assert!(matches!(error, Error::QuotaExceeded { limit: 1, .. }), "{error:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_quota_read_allows_write_after_quota_is_cleared() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "authoritative-quota-quota-cleared";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", br#"{"quota":1}"#.to_vec())
|
||||
.await
|
||||
.expect("set quota");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", Vec::new())
|
||||
.await
|
||||
.expect("clear quota");
|
||||
|
||||
let mut data = PutObjReader::from_vec(b"allowed after clear".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("authoritatively cleared quota should allow writes");
|
||||
|
||||
let mut reader = store
|
||||
.get_object_reader(bucket, "object", None, Default::default(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("read object written after quota clear");
|
||||
let mut bytes = Vec::new();
|
||||
reader.stream.read_to_end(&mut bytes).await.expect("read body");
|
||||
assert_eq!(bytes, b"allowed after clear");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_quota_read_fails_closed_on_invalid_quota_json() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "authoritative-quota-invalid-quota";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
store
|
||||
.update_bucket_metadata_config(bucket, "quota.json", b"{not-json".to_vec())
|
||||
.await
|
||||
.expect("persist invalid quota bytes for fail-closed read");
|
||||
|
||||
let mut data = PutObjReader::from_vec(b"must fail".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("invalid quota JSON must not degrade to no-quota");
|
||||
}
|
||||
|
||||
async fn quota_snapshot_options(store: &Arc<ECStore>, bucket: &str) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
versioned: true,
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
expected_bucket_incarnation_id: Some(store.bucket_incarnation_id(bucket).await.expect("load incarnation")),
|
||||
object_lock_config_snapshot: Some(store.object_lock_config_snapshot(bucket).await.expect("capture snapshot")),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_put_releases_shared_metadata_guard() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -61,6 +61,17 @@ pub(crate) mod minio_generated_read {
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod cleanup_isolation {
|
||||
pub(crate) use super::{DiskAPI, DiskOption, Endpoint, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DeleteOptions, DiskError};
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "test-util", target_os = "linux"))]
|
||||
pub(crate) mod fd_relative {
|
||||
pub(crate) use super::{DiskAPI, DiskOption, DiskStore, Endpoint, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskError, LocalPublicationPause, LocalPublicationStage, ReadOptions};
|
||||
}
|
||||
|
||||
pub(crate) mod pinned_erasure_fixtures {
|
||||
pub(crate) use super::storage_contracts::{BucketOperations, HTTPRangeSpec, MakeBucketOptions, ObjectIO};
|
||||
pub(crate) use super::{ECStore, Endpoint, Erasure, Error, ObjectOptions};
|
||||
|
||||
@@ -90,6 +90,9 @@ pub enum Error {
|
||||
source: Box<Error>,
|
||||
},
|
||||
|
||||
#[error("Bucket discovery incomplete for the requested heal scope; not all buckets could be traversed")]
|
||||
BucketDiscoveryIncomplete,
|
||||
|
||||
#[error("Invalid heal type: {heal_type}")]
|
||||
InvalidHealType { heal_type: String },
|
||||
|
||||
@@ -127,7 +130,7 @@ impl Error {
|
||||
match self {
|
||||
Error::TaskCancelled | Error::TaskTimeout | Error::StaleBucketIncarnation { .. } => false,
|
||||
Error::ReplacementTargetNotReady(_) | Error::DanglingDeleteDeferred { .. } => true,
|
||||
Error::TransientSkip { .. } => true,
|
||||
Error::TransientSkip { .. } | Error::BucketDiscoveryIncomplete => true,
|
||||
// Lock failures classify by LockError's own taxonomy: only the
|
||||
// fatal variants (ResourceNotFound / PermissionDenied /
|
||||
// Configuration) are terminal - retrying cannot fix them - while
|
||||
|
||||
@@ -853,6 +853,13 @@ mod tests {
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> crate::Result<Option<crate::heal::storage_api::status::BucketInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
_selector: Option<crate::heal::storage_api::storage::DiskSetSelector>,
|
||||
) -> crate::Result<Vec<crate::heal::storage_api::status::BucketInfo>> {
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> crate::Result<Vec<crate::heal::storage_api::status::BucketInfo>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
@@ -1400,6 +1400,15 @@ impl ErasureSetHealer {
|
||||
None => self.storage.heal_bucket(bucket, &bucket_opts).await?,
|
||||
};
|
||||
task.record_result_item(result).await;
|
||||
// Keep recovery in the checkpoint's admitted incarnation and repair
|
||||
// the publication bucket before adopting a survivor's delete proof.
|
||||
if let Some(result) = self
|
||||
.storage
|
||||
.recover_delete_transactions(bucket, None, incarnation, &self.heal_opts)
|
||||
.await?
|
||||
{
|
||||
task.record_result_item(result).await;
|
||||
}
|
||||
}
|
||||
|
||||
// 1. get bucket info
|
||||
@@ -2390,6 +2399,8 @@ mod resume_loop_tests {
|
||||
#[derive(Default)]
|
||||
struct FakeStorage {
|
||||
admin_disk: Mutex<Option<DiskStore>>,
|
||||
bucket_discovery_incomplete: AtomicBool,
|
||||
bucket_discovery_scopes: Mutex<Vec<Option<crate::heal::storage_api::storage::DiskSetSelector>>>,
|
||||
bucket_pages: Mutex<HashMap<String, Page>>,
|
||||
incarnations: Mutex<HashMap<String, uuid::Uuid>>,
|
||||
receipts: Mutex<HashMap<String, Option<crate::heal::outcome::HealObjectDisposition>>>,
|
||||
@@ -2469,6 +2480,17 @@ mod resume_loop_tests {
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
selector: Option<crate::heal::storage_api::storage::DiskSetSelector>,
|
||||
) -> Result<Vec<BucketInfo>> {
|
||||
self.bucket_discovery_scopes.lock().expect("discovery scopes").push(selector);
|
||||
if self.bucket_discovery_incomplete.load(Ordering::SeqCst) {
|
||||
return Err(Error::BucketDiscoveryIncomplete);
|
||||
}
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
Ok(self
|
||||
.bucket_pages
|
||||
|
||||
@@ -51,9 +51,10 @@ impl ErasureSetHealer {
|
||||
}
|
||||
let resolved;
|
||||
let buckets = if buckets.is_empty() {
|
||||
let (pool_idx, set_idx) = crate::heal::utils::parse_set_disk_id(set_disk_id)?;
|
||||
resolved = self
|
||||
.storage
|
||||
.list_buckets()
|
||||
.list_buckets_for_admin_heal(Some(crate::heal::storage_api::storage::DiskSetSelector::new(pool_idx, set_idx)))
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|bucket| bucket.name)
|
||||
|
||||
@@ -483,3 +483,50 @@ async fn admin_erasure_checkpoint_publication_cannot_undo_acknowledged_cancel()
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!((outcome.counters.processed, outcome.counters.healed), (1, 1));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_bucket_discovery_rejects_incomplete_checkpoint_and_freezes_complete_empty_scope() {
|
||||
let env = make_env().await;
|
||||
env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst);
|
||||
let result = env
|
||||
.healer
|
||||
.prepare_admin_checkpoint(&env.checkpoint, &[], "pool_0_set_0")
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketDiscoveryIncomplete)));
|
||||
assert!(env.checkpoint.get_checkpoint().await.admin.is_none());
|
||||
let reloaded = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
|
||||
.await
|
||||
.expect("reload checkpoint");
|
||||
assert!(reloaded.get_checkpoint().await.admin.is_none(), "incomplete scope must not be persisted");
|
||||
assert_eq!(
|
||||
env.storage.bucket_discovery_scopes.lock().expect("scopes").as_slice(),
|
||||
&[Some(crate::heal::storage_api::storage::DiskSetSelector::new(0, 0))]
|
||||
);
|
||||
|
||||
env.storage.bucket_discovery_incomplete.store(false, Ordering::SeqCst);
|
||||
assert!(
|
||||
env.healer
|
||||
.prepare_admin_checkpoint(&env.checkpoint, &[], "pool_0_set_0")
|
||||
.await
|
||||
.expect("complete empty scope")
|
||||
.is_empty()
|
||||
);
|
||||
env.storage.bucket_discovery_incomplete.store(true, Ordering::SeqCst);
|
||||
objects(&env.storage, "created-after-admission", &["object"]);
|
||||
let reloaded = CheckpointManager::load_from_disk(env.healer.disk.clone(), &env.task_id)
|
||||
.await
|
||||
.expect("reload frozen scope");
|
||||
assert!(
|
||||
env.healer
|
||||
.prepare_admin_checkpoint(&reloaded, &[], "pool_0_set_0")
|
||||
.await
|
||||
.expect("resume frozen scope")
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
env.storage.bucket_discovery_scopes.lock().expect("scopes").len(),
|
||||
2,
|
||||
"resume must not rediscover or expand scope"
|
||||
);
|
||||
assert!(env.storage.calls().is_empty());
|
||||
}
|
||||
|
||||
@@ -13,6 +13,18 @@
|
||||
// limitations under the License.
|
||||
/// The automatic disk scanner: replacement discovery and unformatted-disk enqueue.
|
||||
use super::*;
|
||||
use crate::heal::DiskStore;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
pub(super) fn extend_replacement_recovery_disks(
|
||||
inventory: &mut BTreeMap<String, DiskStore>,
|
||||
disks: impl IntoIterator<Item = DiskStore>,
|
||||
) {
|
||||
for disk in disks {
|
||||
// Preserve the local snapshot when later inventory repeats its endpoint.
|
||||
inventory.entry(disk.endpoint().to_string()).or_insert(disk);
|
||||
}
|
||||
}
|
||||
|
||||
impl HealManager {
|
||||
/// Start background task to auto scan local disks and enqueue erasure set heal requests
|
||||
@@ -87,7 +99,20 @@ impl HealManager {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
local_disk_map.values().flatten().cloned().collect::<Vec<_>>()
|
||||
};
|
||||
let local_endpoints = local_disks.iter().map(|disk| disk.endpoint()).collect::<Vec<_>>();
|
||||
let mut local_endpoints = HashMap::with_capacity(local_disks.len());
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
local_endpoints.entry(endpoint.to_string()).or_insert(endpoint);
|
||||
}
|
||||
// Formatting removes a target from blank-disk discovery, but its
|
||||
// unfinished intent may still live on a remote survivor.
|
||||
let mut recovery_sets = local_disks
|
||||
.iter()
|
||||
.filter_map(|disk| {
|
||||
let endpoint = disk.endpoint();
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx)
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
let blocked_sets = replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
@@ -97,6 +122,7 @@ impl HealManager {
|
||||
let mut retry_succeeded = HashSet::new();
|
||||
let mut retry_failed = HashSet::new();
|
||||
for set_disk_id in blocked_sets {
|
||||
recovery_sets.remove(&set_disk_id);
|
||||
let disks = match storage.replacement_intent_disks(&set_disk_id).await {
|
||||
Ok(disks) => disks,
|
||||
Err(error) => {
|
||||
@@ -235,11 +261,12 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
|
||||
let mut recovery_disks = local_disks.clone();
|
||||
recovery_disks.extend(retry_recovery_disks);
|
||||
for set_disk_id in endpoints.keys() {
|
||||
let mut recovery_disks = BTreeMap::new();
|
||||
extend_replacement_recovery_disks(&mut recovery_disks, local_disks.iter().cloned());
|
||||
extend_replacement_recovery_disks(&mut recovery_disks, retry_recovery_disks);
|
||||
for set_disk_id in &recovery_sets {
|
||||
match storage.replacement_intent_disks(set_disk_id).await {
|
||||
Ok(disks) => recovery_disks.extend(disks),
|
||||
Ok(disks) => extend_replacement_recovery_disks(&mut recovery_disks, disks),
|
||||
Err(error) => {
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
warn!(
|
||||
@@ -254,10 +281,6 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
recovery_disks.sort_by_key(|disk| disk.endpoint().to_string());
|
||||
recovery_disks
|
||||
.dedup_by(|left, right| left.endpoint().to_string() == right.endpoint().to_string());
|
||||
|
||||
// Once formatting succeeds a replacement is no longer
|
||||
// discoverable as UnformattedDisk. Re-admit exactly one
|
||||
// incomplete durable generation per set within its
|
||||
@@ -265,7 +288,7 @@ impl HealManager {
|
||||
// verified terminal cleanup. Multiple generations are a
|
||||
// durable conflict: leave every marker/state intact and
|
||||
// require reconciliation rather than choosing one.
|
||||
for disk in &recovery_disks {
|
||||
for (anchor, disk) in &recovery_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
let disk_set_disk_id =
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
|
||||
@@ -351,7 +374,7 @@ impl HealManager {
|
||||
// A remote survivor may own the durable intent, but only the
|
||||
// node mounting every replacement target may advance or replay it.
|
||||
if state.replacement_targets.iter().any(|target| {
|
||||
!local_endpoints.iter().any(|endpoint| endpoint.to_string() == *target)
|
||||
!local_endpoints.contains_key(target)
|
||||
}) {
|
||||
deferred_recovery_sets.insert(state.set_disk_id.clone());
|
||||
continue;
|
||||
@@ -361,19 +384,20 @@ impl HealManager {
|
||||
}
|
||||
let mut selected = durable_recovery_candidates.remove(&task_id);
|
||||
let previous_set = selected.as_ref().map(|candidate| candidate.state.set_disk_id.clone());
|
||||
let candidate_set = state.set_disk_id.clone();
|
||||
let candidate = match ReplacementRecoveryCandidate::new(
|
||||
state.clone(),
|
||||
disk.endpoint().to_string(),
|
||||
disk.endpoint().is_local,
|
||||
state,
|
||||
anchor.clone(),
|
||||
endpoint.is_local,
|
||||
) {
|
||||
Ok(candidate) => candidate,
|
||||
Err(error) => {
|
||||
conflicted_recovery_tasks.insert(task_id.clone());
|
||||
conflicted_recovery_sets.insert(state.set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(candidate_set.clone());
|
||||
if let Some(previous_set) = &previous_set {
|
||||
conflicted_recovery_sets.insert(previous_set.clone());
|
||||
}
|
||||
block_recovery_set(&state.set_disk_id);
|
||||
block_recovery_set(&candidate_set);
|
||||
if let Some(previous_set) = &previous_set {
|
||||
block_recovery_set(previous_set);
|
||||
}
|
||||
@@ -432,12 +456,7 @@ impl HealManager {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let Some(anchor_disk) = local_disks
|
||||
.iter()
|
||||
.chain(recovery_disks.iter())
|
||||
.find(|disk| disk.endpoint().to_string() == candidate.anchor)
|
||||
.cloned()
|
||||
else {
|
||||
let Some(anchor_disk) = recovery_disks.get(&candidate.anchor).cloned() else {
|
||||
deferred_recovery_sets.insert(candidate.state.set_disk_id.clone());
|
||||
continue;
|
||||
};
|
||||
@@ -530,12 +549,7 @@ impl HealManager {
|
||||
let targets = state
|
||||
.replacement_targets
|
||||
.iter()
|
||||
.filter_map(|target| {
|
||||
local_endpoints
|
||||
.iter()
|
||||
.find(|endpoint| endpoint.to_string() == *target)
|
||||
.cloned()
|
||||
})
|
||||
.filter_map(|target| local_endpoints.get(target).cloned())
|
||||
.collect::<Vec<_>>();
|
||||
if targets.len() != state.replacement_targets.len() {
|
||||
deferred_recovery_sets.insert(state.set_disk_id.clone());
|
||||
@@ -607,20 +621,9 @@ impl HealManager {
|
||||
}
|
||||
|
||||
for (set_disk_id, (_, targets, _, _)) in &durable_recoveries {
|
||||
let expected = targets.iter().map(ToString::to_string).collect::<HashSet<_>>();
|
||||
let observed = endpoints
|
||||
.get(set_disk_id)
|
||||
.map(|endpoints| endpoints.iter().map(ToString::to_string).collect::<HashSet<_>>())
|
||||
.unwrap_or_default();
|
||||
if !observed.is_subset(&expected) {
|
||||
replacement_recovery_blocked_sets
|
||||
.lock()
|
||||
.expect("replacement recovery blocked set lock poisoned")
|
||||
.insert(set_disk_id.clone());
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
continue;
|
||||
}
|
||||
endpoints.entry(set_disk_id.clone()).or_default().extend(targets.clone());
|
||||
// The durable owner must finish its exact target scope before
|
||||
// newly discovered replacements can start another generation.
|
||||
endpoints.insert(set_disk_id.clone(), targets.clone());
|
||||
}
|
||||
for set_disk_id in &conflicted_recovery_sets {
|
||||
durable_recoveries.remove(set_disk_id);
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
//! or deletion.
|
||||
|
||||
use super::*;
|
||||
use crate::heal::outcome::HealExecutionOutcome;
|
||||
use crate::heal::resume::CheckpointManager;
|
||||
use crate::heal::storage_api::owner::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes};
|
||||
use crate::heal::{DiskStore, RUSTFS_META_BUCKET};
|
||||
@@ -36,12 +37,15 @@ const ROOT_RECOVERY_PREFIX: &str = "root-heal-";
|
||||
const ROOT_TERMINAL_PREFIX: &str = "terminal-root-heal-";
|
||||
const LEGACY_ROOT_RECOVERY_SCHEMA: u32 = 1;
|
||||
const SCOPED_ROOT_RECOVERY_SCHEMA: u32 = 2;
|
||||
const ROOT_RECOVERY_SCHEMA: u32 = 3;
|
||||
const BUCKET_ROOT_RECOVERY_SCHEMA: u32 = 3;
|
||||
const ROOT_RECOVERY_SCHEMA: u32 = 4;
|
||||
const ROOT_TERMINAL_SCHEMA: u32 = 1;
|
||||
const ROOT_QUARANTINE_INTENT_PREFIX: &str = "quarantined-root-heal-intent-";
|
||||
const ROOT_QUARANTINE_TERMINAL_PREFIX: &str = "quarantined-root-heal-terminal-";
|
||||
const ROOT_QUARANTINE_SCHEMA: u32 = 1;
|
||||
pub(super) const MAX_ROOT_RECOVERY_BYTES: usize = 64 * 1024;
|
||||
// Cluster checkpoints additionally retain a frozen inventory and the bounded outcome window.
|
||||
pub(super) const MAX_CLUSTER_RECOVERY_BYTES: usize = 16 * 1024 * 1024;
|
||||
const ROOT_TERMINAL_GC_SCAN_BUDGET: usize = 1024;
|
||||
const ROOT_TERMINAL_GC_DELETE_BUDGET: usize = 64;
|
||||
|
||||
@@ -237,7 +241,83 @@ fn default_recovery_heal_type() -> RecoveryHealType {
|
||||
RecoveryHealType::Cluster
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct AdminClusterScope {
|
||||
#[serde(deserialize_with = "decode_cluster_buckets")]
|
||||
pub(crate) buckets: std::collections::BTreeMap<String, Uuid>,
|
||||
pub(crate) next_bucket_index: usize,
|
||||
pub(crate) outcome: HealTaskOutcome,
|
||||
pub(crate) progress: HealProgress,
|
||||
}
|
||||
|
||||
fn decode_cluster_buckets<'de, D: serde::Deserializer<'de>>(
|
||||
deserializer: D,
|
||||
) -> std::result::Result<std::collections::BTreeMap<String, Uuid>, D::Error> {
|
||||
struct Buckets;
|
||||
impl<'de> serde::de::Visitor<'de> for Buckets {
|
||||
type Value = std::collections::BTreeMap<String, Uuid>;
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
formatter.write_str("unique bucket generation bindings")
|
||||
}
|
||||
fn visit_map<M: serde::de::MapAccess<'de>>(self, mut map: M) -> std::result::Result<Self::Value, M::Error> {
|
||||
let mut buckets = std::collections::BTreeMap::new();
|
||||
while let Some((bucket, incarnation)) = map.next_entry::<String, Uuid>()? {
|
||||
if buckets.insert(bucket, incarnation).is_some() {
|
||||
return Err(serde::de::Error::custom("duplicate administrator cluster bucket"));
|
||||
}
|
||||
}
|
||||
Ok(buckets)
|
||||
}
|
||||
}
|
||||
deserializer.deserialize_map(Buckets)
|
||||
}
|
||||
|
||||
impl AdminClusterScope {
|
||||
pub(crate) fn checkpoint_interval(&self) -> usize {
|
||||
// Cap full-inventory rewrites per traversal, including many empty buckets.
|
||||
self.buckets.len().div_ceil(32).max(1)
|
||||
}
|
||||
|
||||
fn validate(&self) -> Result<()> {
|
||||
if self.next_bucket_index > self.buckets.len()
|
||||
|| self.outcome.execution != HealExecutionOutcome::Running
|
||||
|| self.outcome.coverage != crate::heal::outcome::HealTraversalCoverage::Partial
|
||||
|| self.outcome.counters.failed != 0
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint("Invalid administrator cluster checkpoint progress".to_string()));
|
||||
}
|
||||
for (bucket, incarnation) in &self.buckets {
|
||||
if bucket.contains('/')
|
||||
|| bucket.trim() != bucket
|
||||
|| crate::heal::storage_api::owner::ecstore_is_reserved_or_invalid_bucket(bucket, false)
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint("Invalid administrator cluster bucket name".to_string()));
|
||||
}
|
||||
if incarnation.is_nil() {
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"Administrator cluster checkpoint lacks a bucket incarnation".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let completed_through = self
|
||||
.next_bucket_index
|
||||
.checked_sub(1)
|
||||
.and_then(|index| self.buckets.keys().nth(index));
|
||||
if self.outcome.objects.iter().any(|item| {
|
||||
completed_through.is_none_or(|last| item.identity.bucket > *last)
|
||||
|| self.buckets.get(&item.identity.bucket).copied() != item.identity.bucket_incarnation_id
|
||||
}) || (self.next_bucket_index == 0 && self.outcome.counters.processed != 0)
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"Administrator cluster checkpoint outcome is outside its completed scope".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct RootHealIntent {
|
||||
schema: u32,
|
||||
@@ -246,6 +326,8 @@ struct RootHealIntent {
|
||||
heal_type: RecoveryHealType,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
bucket_incarnation_id: Option<Uuid>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
cluster_scope: Option<AdminClusterScope>,
|
||||
#[serde(deserialize_with = "decode_options")]
|
||||
options: HealOptions,
|
||||
priority: HealPriority,
|
||||
@@ -292,6 +374,26 @@ struct RootHealSource {
|
||||
source_oversized: bool,
|
||||
}
|
||||
|
||||
impl RootHealQuarantine {
|
||||
fn matches_source(&self, source: &RootHealSource) -> bool {
|
||||
if self.source_prefix_len == source.bytes.len()
|
||||
&& self.source_prefix_sha256 == source.source_prefix_sha256
|
||||
&& self.source_oversized == source.source_oversized
|
||||
{
|
||||
return true;
|
||||
}
|
||||
// Before schema 4, oversized intents retained only the 64 KiB prefix.
|
||||
// Keep that quarantine fence across upgrades without reinterpreting its source.
|
||||
self.record_kind == RootHealRecordKind::Intent
|
||||
&& self.source_oversized
|
||||
&& self.source_prefix_len == MAX_ROOT_RECOVERY_BYTES + 1
|
||||
&& source
|
||||
.bytes
|
||||
.get(..self.source_prefix_len)
|
||||
.is_some_and(|prefix| Sha256::digest(prefix).as_slice() == self.source_prefix_sha256.as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub(super) struct RootHealInventory {
|
||||
pub(super) requests: Vec<HealRequest>,
|
||||
@@ -362,12 +464,38 @@ impl RootHealTerminal {
|
||||
}
|
||||
|
||||
impl RootHealIntent {
|
||||
fn encode(&self) -> Result<EcstoreDiskBytes> {
|
||||
let limit = match &self.cluster_scope {
|
||||
// Reserve headroom for the bounded outcome window and progress so
|
||||
// completing work cannot overflow an inventory admitted at the limit.
|
||||
Some(scope) if scope.next_bucket_index == 0 => MAX_CLUSTER_RECOVERY_BYTES - 1024 * 1024,
|
||||
Some(_) => MAX_CLUSTER_RECOVERY_BYTES,
|
||||
None => MAX_ROOT_RECOVERY_BYTES,
|
||||
};
|
||||
encode_root_record(self, limit)
|
||||
}
|
||||
|
||||
fn validate_task(&self, task: &HealTask) -> Result<()> {
|
||||
let mut options = self.options.clone();
|
||||
options.timeout = task.options.timeout;
|
||||
if HealType::from(self.heal_type.clone()) != task.heal_type
|
||||
|| self.bucket_incarnation_id != task.bucket_incarnation_id
|
||||
|| self.created_at != task.created_at
|
||||
|| self.priority != task.priority
|
||||
|| options != task.options
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint(format!("Root heal recovery owner changed for {}", task.id)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn from_request(request: &HealRequest) -> Result<Self> {
|
||||
Ok(Self {
|
||||
schema: ROOT_RECOVERY_SCHEMA,
|
||||
task_id: request.id.clone(),
|
||||
heal_type: RecoveryHealType::try_from(&request.heal_type)?,
|
||||
bucket_incarnation_id: request.bucket_incarnation_id,
|
||||
cluster_scope: None,
|
||||
options: request.options.clone(),
|
||||
priority: request.priority,
|
||||
retry_attempts: request.retry_attempts,
|
||||
@@ -386,6 +514,32 @@ impl RootHealIntent {
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_root_record(value: &impl Serialize, limit: usize) -> Result<EcstoreDiskBytes> {
|
||||
struct BoundedWriter {
|
||||
bytes: Vec<u8>,
|
||||
limit: usize,
|
||||
}
|
||||
impl std::io::Write for BoundedWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
if bytes.len() > self.limit.saturating_sub(self.bytes.len()) {
|
||||
return Err(std::io::Error::other("root heal record exceeds its size limit"));
|
||||
}
|
||||
self.bytes.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut writer = BoundedWriter {
|
||||
bytes: Vec::new(),
|
||||
limit,
|
||||
};
|
||||
serde_json::to_writer(&mut writer, value)
|
||||
.map_err(|error| Error::Serialization(format!("Serialize root heal record: {error}")))?;
|
||||
Ok(writer.bytes.into())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct RootHealRecovery {
|
||||
mutation: Mutex<()>,
|
||||
@@ -490,19 +644,27 @@ async fn read_bounded(disk: &DiskStore, path: &str) -> Result<Option<EcstoreDisk
|
||||
};
|
||||
let mut bytes = Vec::new();
|
||||
reader
|
||||
.take(u64::try_from(MAX_ROOT_RECOVERY_BYTES + 1).map_err(Error::other)?)
|
||||
.take(u64::try_from(root_record_read_limit(path) + 1).map_err(Error::other)?)
|
||||
.read_to_end(&mut bytes)
|
||||
.await?;
|
||||
Ok(Some(bytes.into()))
|
||||
}
|
||||
|
||||
fn root_record_read_limit(path: &str) -> usize {
|
||||
if path.starts_with(ROOT_RECOVERY_PREFIX) {
|
||||
MAX_CLUSTER_RECOVERY_BYTES
|
||||
} else {
|
||||
MAX_ROOT_RECOVERY_BYTES
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_source(disk: &DiskStore, path: &str) -> Result<Option<RootHealSource>> {
|
||||
let Some(bytes) = read_bounded(disk, path).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
Ok(Some(RootHealSource {
|
||||
source_prefix_sha256: Sha256::digest(&bytes).to_vec(),
|
||||
source_oversized: bytes.len() > MAX_ROOT_RECOVERY_BYTES,
|
||||
source_oversized: bytes.len() > root_record_read_limit(path),
|
||||
bytes,
|
||||
}))
|
||||
}
|
||||
@@ -544,7 +706,7 @@ fn encode_quarantine(
|
||||
|
||||
fn decode_intent(task_id: &str, bytes: &[u8]) -> Result<RootHealIntent> {
|
||||
let _ = intent_path(task_id)?;
|
||||
if bytes.len() > MAX_ROOT_RECOVERY_BYTES {
|
||||
if bytes.len() > MAX_CLUSTER_RECOVERY_BYTES {
|
||||
return Err(Error::Other(format!("Root heal recovery record exceeds its size limit {task_id}")));
|
||||
}
|
||||
let mut intent: RootHealIntent = serde_json::from_slice(bytes)
|
||||
@@ -556,9 +718,25 @@ fn decode_intent(task_id: &str, bytes: &[u8]) -> Result<RootHealIntent> {
|
||||
LEGACY_ROOT_RECOVERY_SCHEMA
|
||||
if intent.heal_type == RecoveryHealType::Cluster && intent.bucket_incarnation_id.is_none() => {}
|
||||
SCOPED_ROOT_RECOVERY_SCHEMA if intent.bucket_incarnation_id.is_none() => {}
|
||||
BUCKET_ROOT_RECOVERY_SCHEMA if intent.cluster_scope.is_none() => {}
|
||||
ROOT_RECOVERY_SCHEMA => {}
|
||||
_ => return Err(Error::Other(format!("Unsupported or mismatched root heal recovery record {task_id}"))),
|
||||
}
|
||||
if intent.heal_type == RecoveryHealType::Cluster && intent.schema != ROOT_RECOVERY_SCHEMA {
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"Legacy cluster heal has no frozen bucket generations; start a fresh task".to_string(),
|
||||
));
|
||||
}
|
||||
match &intent.cluster_scope {
|
||||
Some(scope) if intent.schema == ROOT_RECOVERY_SCHEMA && intent.heal_type == RecoveryHealType::Cluster => {
|
||||
scope.validate()?;
|
||||
}
|
||||
Some(_) => return Err(Error::InvalidCheckpoint("Unexpected administrator cluster scope".to_string())),
|
||||
None if bytes.len() > MAX_ROOT_RECOVERY_BYTES => {
|
||||
return Err(Error::InvalidCheckpoint("Root heal recovery record exceeds its size limit".to_string()));
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
if !matches!(intent.heal_type, RecoveryHealType::Bucket { .. }) && intent.bucket_incarnation_id.is_some() {
|
||||
return Err(Error::Other(format!(
|
||||
"Unexpected bucket incarnation in root heal recovery record {task_id}"
|
||||
@@ -723,10 +901,7 @@ impl RootHealRecovery {
|
||||
}
|
||||
};
|
||||
let marker = decode_quarantine(task_id, record_kind, &marker_bytes)?;
|
||||
if marker.source_prefix_len != source.bytes.len()
|
||||
|| marker.source_prefix_sha256 != source.source_prefix_sha256
|
||||
|| marker.source_oversized != source.source_oversized
|
||||
{
|
||||
if !marker.matches_source(source) {
|
||||
return Err(Error::other(format!("Root heal quarantine source changed {task_id}")));
|
||||
}
|
||||
Ok(marker)
|
||||
@@ -808,9 +983,7 @@ impl RootHealRecovery {
|
||||
if let Some((owner, source)) = terminal_raw.into_iter().next() {
|
||||
if let Some(marker) = markers.get(&(RootHealRecordKind::Terminal, task_id.clone())) {
|
||||
if EcstoreDiskAPI::endpoint(marker.0.as_ref()) != EcstoreDiskAPI::endpoint(owner.as_ref())
|
||||
|| marker.1.source_prefix_len != source.bytes.len()
|
||||
|| marker.1.source_prefix_sha256 != source.source_prefix_sha256
|
||||
|| marker.1.source_oversized != source.source_oversized
|
||||
|| !marker.1.matches_source(&source)
|
||||
{
|
||||
return Err(Error::other(format!("Root heal quarantine source changed {task_id}")));
|
||||
}
|
||||
@@ -849,9 +1022,7 @@ impl RootHealRecovery {
|
||||
};
|
||||
if let Some(marker) = markers.get(&(RootHealRecordKind::Intent, task_id.clone())) {
|
||||
if EcstoreDiskAPI::endpoint(marker.0.as_ref()) != EcstoreDiskAPI::endpoint(owner.as_ref())
|
||||
|| marker.1.source_prefix_len != source.bytes.len()
|
||||
|| marker.1.source_prefix_sha256 != source.source_prefix_sha256
|
||||
|| marker.1.source_oversized != source.source_oversized
|
||||
|| !marker.1.matches_source(&source)
|
||||
{
|
||||
return Err(Error::other(format!("Root heal quarantine source changed {task_id}")));
|
||||
}
|
||||
@@ -924,18 +1095,29 @@ impl RootHealRecovery {
|
||||
if request.options.no_lock {
|
||||
return Err(Error::Other("Administrator root heal cannot skip namespace locking".to_string()));
|
||||
}
|
||||
let intent = RootHealIntent::from_request(request)?;
|
||||
let mut intent = RootHealIntent::from_request(request)?;
|
||||
intent.heal_type.validate()?;
|
||||
let bytes =
|
||||
serde_json::to_vec(&intent).map_err(|error| Error::Other(format!("Serialize root heal recovery record: {error}")))?;
|
||||
let path = intent_path(&request.id)?;
|
||||
if let Some((disk, expected)) = existing {
|
||||
let previous = decode_intent(&request.id, &expected)?;
|
||||
let mut previous_options = previous.options.clone();
|
||||
previous_options.timeout = intent.options.timeout;
|
||||
if previous.heal_type != intent.heal_type
|
||||
|| previous.bucket_incarnation_id != intent.bucket_incarnation_id
|
||||
|| previous.created_at != intent.created_at
|
||||
|| previous.priority != intent.priority
|
||||
|| previous_options != intent.options
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint("Root heal recovery owner changed during handoff".to_string()));
|
||||
}
|
||||
intent.cluster_scope = previous.cluster_scope;
|
||||
let bytes = intent.encode()?;
|
||||
return match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
Some(expected),
|
||||
Some(bytes.into()),
|
||||
Some(bytes),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -944,19 +1126,15 @@ impl RootHealRecovery {
|
||||
};
|
||||
}
|
||||
|
||||
let bytes = intent.encode()?;
|
||||
|
||||
if disks.is_empty() {
|
||||
return Err(Error::Other("No local disk available for root heal shutdown recovery".to_string()));
|
||||
}
|
||||
let mut last_not_committed = None;
|
||||
for disk in &disks {
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
None,
|
||||
Some(bytes.clone().into()),
|
||||
)
|
||||
.await
|
||||
match EcstoreDiskAPI::compare_and_update_file(disk.as_ref(), RUSTFS_META_BUCKET, &path, None, Some(bytes.clone()))
|
||||
.await
|
||||
{
|
||||
Ok(EcstoreConditionalFileUpdate::Updated) => return Ok(()),
|
||||
Ok(_) => return Err(Error::Other(format!("Root heal recovery record changed for {}", request.id))),
|
||||
@@ -970,6 +1148,63 @@ impl RootHealRecovery {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn cluster_scope(&self, task: &HealTask) -> Result<Option<AdminClusterScope>> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(None);
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let (_, bytes) = Self::find(&self.disks().await?, &task.id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::InvalidCheckpoint("Administrator cluster intent is missing".to_string()))?;
|
||||
let intent = decode_intent(&task.id, &bytes)?;
|
||||
intent.validate_task(task)?;
|
||||
Ok(intent.cluster_scope)
|
||||
}
|
||||
|
||||
pub(crate) async fn checkpoint_cluster_scope(&self, task: &HealTask, scope: &AdminClusterScope) -> Result<()> {
|
||||
scope.validate()?;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if self.disabled_for_tests {
|
||||
return Ok(());
|
||||
}
|
||||
let _guard = self.mutation.lock().await;
|
||||
let (disk, expected) = Self::find(&self.disks().await?, &task.id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::InvalidCheckpoint("Administrator cluster intent is missing".to_string()))?;
|
||||
let mut intent = decode_intent(&task.id, &expected)?;
|
||||
intent.validate_task(task)?;
|
||||
if intent.heal_type != RecoveryHealType::Cluster {
|
||||
return Err(Error::InvalidCheckpoint("Cluster scope requires a cluster task".to_string()));
|
||||
}
|
||||
if let Some(previous) = &intent.cluster_scope {
|
||||
if previous.buckets != scope.buckets
|
||||
|| scope.next_bucket_index < previous.next_bucket_index
|
||||
|| scope.next_bucket_index > previous.next_bucket_index.saturating_add(scope.checkpoint_interval())
|
||||
{
|
||||
return Err(Error::InvalidCheckpoint(
|
||||
"Administrator cluster scope changed during recovery".to_string(),
|
||||
));
|
||||
}
|
||||
} else if scope.next_bucket_index != 0 {
|
||||
return Err(Error::InvalidCheckpoint("Administrator cluster scope was not initialized".to_string()));
|
||||
}
|
||||
intent.cluster_scope = Some(scope.clone());
|
||||
let bytes = intent.encode()?;
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(&task.id)?,
|
||||
Some(expected),
|
||||
Some(bytes),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
EcstoreConditionalFileUpdate::Updated => Ok(()),
|
||||
_ => Err(Error::InvalidCheckpoint("Administrator cluster checkpoint owner changed".to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn remove(&self, task_id: &str, heal_type: &HealType, source: HealRequestSource) -> Result<bool> {
|
||||
if !is_admin_heal_recovery(heal_type, source) {
|
||||
return Ok(false);
|
||||
@@ -1015,16 +1250,7 @@ impl RootHealRecovery {
|
||||
return Ok(());
|
||||
};
|
||||
let mut intent = decode_intent(&task.id, &expected)?;
|
||||
if HealType::from(intent.heal_type.clone()) != task.heal_type
|
||||
|| intent.bucket_incarnation_id != task.bucket_incarnation_id
|
||||
{
|
||||
return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id)));
|
||||
}
|
||||
let mut expected_options = intent.options.clone();
|
||||
expected_options.timeout = task.options.timeout;
|
||||
if intent.created_at != task.created_at || intent.priority != task.priority || expected_options != task.options {
|
||||
return Err(Error::Other(format!("Root heal recovery owner changed for {}", task.id)));
|
||||
}
|
||||
intent.validate_task(task)?;
|
||||
// A terminal timeout leaves no runtime owner for stop() to snapshot.
|
||||
// Checkpoint its consumed budget before publishing terminal status;
|
||||
// never refund time if an earlier checkpoint is already stricter.
|
||||
@@ -1033,14 +1259,13 @@ impl RootHealRecovery {
|
||||
(previous, remaining) => previous.or(remaining),
|
||||
};
|
||||
intent.retry_attempts = intent.retry_attempts.max(task.retry_attempts);
|
||||
let bytes = serde_json::to_vec(&intent)
|
||||
.map_err(|error| Error::Other(format!("Serialize root heal recovery checkpoint: {error}")))?;
|
||||
let bytes = intent.encode()?;
|
||||
match EcstoreDiskAPI::compare_and_update_file(
|
||||
disk.as_ref(),
|
||||
RUSTFS_META_BUCKET,
|
||||
&intent_path(&task.id)?,
|
||||
Some(expected),
|
||||
Some(bytes.into()),
|
||||
Some(bytes),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
|
||||
use super::*;
|
||||
use crate::heal::outcome::{HealAbortReason, HealExecutionOutcome};
|
||||
use std::io::Write;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
// This prefix must not overlap either namespace understood by schema-1 readers.
|
||||
@@ -68,24 +67,7 @@ impl RootHealReport {
|
||||
}
|
||||
|
||||
pub(super) fn encode(&self) -> Result<EcstoreDiskBytes> {
|
||||
struct BoundedWriter(Vec<u8>);
|
||||
impl Write for BoundedWriter {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
if bytes.len() > MAX_ROOT_REPORT_BYTES.saturating_sub(self.0.len()) {
|
||||
return Err(std::io::Error::other("root heal report exceeds its size limit"));
|
||||
}
|
||||
self.0.extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut writer = BoundedWriter(Vec::new());
|
||||
serde_json::to_writer(&mut writer, self)
|
||||
.map_err(|error| Error::Serialization(format!("Serialize root heal report: {error}")))?;
|
||||
Ok(writer.0.into())
|
||||
encode_root_record(self, MAX_ROOT_REPORT_BYTES)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ use tempfile::TempDir;
|
||||
pub(super) mod admin_overlap;
|
||||
mod root_recovery;
|
||||
use uuid::Uuid;
|
||||
mod auto_scan;
|
||||
mod running_mainline;
|
||||
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk, storage_api::status::BucketInfo};
|
||||
@@ -601,6 +602,13 @@ impl HealStorageAPI for MockStorage {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
_selector: Option<crate::heal::storage_api::storage::DiskSetSelector>,
|
||||
) -> Result<Vec<BucketInfo>> {
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
if let Some(hook) = manager_recovery_test_hook() {
|
||||
*hook.listed.lock().expect("manager recovery listed lock should not poison") = true;
|
||||
@@ -5663,7 +5671,7 @@ async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() {
|
||||
process_manager_queue_once(&manager).await;
|
||||
let explicit_status = tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
if let Ok(status @ HealTaskStatus::Failed { .. }) = manager.get_task_status(&explicit_id).await {
|
||||
if let Ok(status @ HealTaskStatus::Completed) = manager.get_task_status(&explicit_id).await {
|
||||
break status;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
@@ -5671,7 +5679,8 @@ async fn configured_task_timeout_applies_only_when_request_timeout_is_absent() {
|
||||
})
|
||||
.await
|
||||
.expect("explicit timeout request should finish without using the zero default");
|
||||
assert!(matches!(explicit_status, HealTaskStatus::Failed { .. }));
|
||||
// Generation-bound admin tasks reach the successful mock heal without a namespace preflight.
|
||||
assert_eq!(explicit_status, HealTaskStatus::Completed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -5902,3 +5911,25 @@ fn test_heal_config_respects_feature_flags() {
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_bucket_discovery_retries_keep_the_existing_budget() {
|
||||
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
|
||||
let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let id = request.id.clone();
|
||||
let error = Err(Error::BucketDiscoveryIncomplete);
|
||||
for attempt in 1..=MAX_RECOVERABLE_HEAL_RETRIES {
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
let (retry, delay, _) = retry_request_for_result(&task, &error).expect("discovery can recover within budget");
|
||||
assert_eq!(retry.id, id);
|
||||
assert_eq!(retry.retry_attempts, attempt);
|
||||
assert!(delay > Duration::ZERO);
|
||||
request = retry;
|
||||
}
|
||||
let task = HealTask::from_request(request, storage);
|
||||
assert!(
|
||||
retry_request_for_result(&task, &error).is_none(),
|
||||
"persistent incompleteness must terminate"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::heal::storage::HealListItem;
|
||||
use crate::heal::storage_api::{
|
||||
EcstoreEndpointServerPools as EndpointServerPools, EcstoreEndpoints as Endpoints, EcstorePoolEndpoints as PoolEndpoints,
|
||||
ecstore_init_local_disks as init_local_disks,
|
||||
};
|
||||
use std::{
|
||||
future::Future,
|
||||
sync::atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
const SET: &str = "pool_0_set_0";
|
||||
const SCAN_INTERVAL: Duration = Duration::from_secs(10);
|
||||
|
||||
struct ScannerStorage {
|
||||
survivor: DiskStore,
|
||||
identity: ReplacementTargetIdentity,
|
||||
inventory_calls: AtomicUsize,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl HealStorageAPI for ScannerStorage {
|
||||
async fn replacement_intent_disks(&self, set: &str) -> Result<Vec<DiskStore>> {
|
||||
assert_eq!(set, SET, "discovery must use the target's erasure set");
|
||||
self.inventory_calls.fetch_add(1, Ordering::SeqCst);
|
||||
// This disk is deliberately outside the node's local registry. The
|
||||
// mock supplies the same inventory boundary as a remote survivor,
|
||||
// while real disk I/O exercises durable intent discovery and loading.
|
||||
Ok(vec![self.survivor.clone()])
|
||||
}
|
||||
|
||||
async fn replacement_target_identities(&self, targets: &[String]) -> Result<Vec<ReplacementTargetIdentity>> {
|
||||
assert_eq!(targets, std::slice::from_ref(&self.identity.endpoint));
|
||||
Ok(vec![self.identity.clone()])
|
||||
}
|
||||
|
||||
async fn get_object_meta(&self, _: &str, _: &str) -> Result<Option<HealObjectInfo>> {
|
||||
Err(Error::other("scanner admission must not inspect objects"))
|
||||
}
|
||||
|
||||
async fn ec_decode_rebuild(&self, _: &str, _: &str) -> Result<Vec<u8>> {
|
||||
Err(Error::other("scanner admission must not rebuild objects"))
|
||||
}
|
||||
|
||||
async fn get_bucket_info(&self, _: &str) -> Result<Option<BucketInfo>> {
|
||||
Err(Error::other("scanner admission must not inspect buckets"))
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
Err(Error::other("same-incarnation discovery must reuse durable buckets"))
|
||||
}
|
||||
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
_: Option<crate::heal::storage_api::storage::DiskSetSelector>,
|
||||
) -> Result<Vec<BucketInfo>> {
|
||||
Err(Error::other("scanner admission must not start admin discovery"))
|
||||
}
|
||||
|
||||
async fn object_exists(&self, _: &str, _: &str) -> Result<bool> {
|
||||
Err(Error::other("scanner admission must not inspect objects"))
|
||||
}
|
||||
|
||||
async fn heal_object(&self, _: &str, _: &str, _: Option<&str>, _: &HealOpts) -> Result<(HealResultItem, Option<Error>)> {
|
||||
Err(Error::other("scanner admission must not execute object healing"))
|
||||
}
|
||||
|
||||
async fn heal_bucket(&self, _: &str, _: &HealOpts) -> Result<HealResultItem> {
|
||||
Err(Error::other("scanner admission must not execute bucket healing"))
|
||||
}
|
||||
|
||||
async fn heal_format(&self, _: bool) -> Result<(HealResultItem, Option<Error>)> {
|
||||
Err(Error::other("scanner admission must not execute format healing"))
|
||||
}
|
||||
|
||||
async fn list_objects_for_heal_page(
|
||||
&self,
|
||||
_: &str,
|
||||
_: &str,
|
||||
_: Option<&str>,
|
||||
_: bool,
|
||||
) -> Result<(Vec<HealListItem>, Option<String>, bool)> {
|
||||
Err(Error::other("scanner admission must not traverse objects"))
|
||||
}
|
||||
|
||||
async fn get_disk_for_resume(&self, _: &str) -> Result<DiskStore> {
|
||||
Err(Error::other("scanner admission must retain the durable survivor"))
|
||||
}
|
||||
}
|
||||
|
||||
struct ScannerFixture {
|
||||
_temp: TempDir,
|
||||
manager: HealManager,
|
||||
storage: Arc<ScannerStorage>,
|
||||
owner: ResumeManager,
|
||||
task_id: String,
|
||||
local_targets: Vec<String>,
|
||||
}
|
||||
|
||||
impl Drop for ScannerFixture {
|
||||
fn drop(&mut self) {
|
||||
self.manager.cancel_token.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
impl ScannerFixture {
|
||||
async fn new(unformatted_siblings: bool) -> Self {
|
||||
let temp = TempDir::new().expect("scanner replacement fixture directory");
|
||||
let disk_ids = (0..3).map(|_| Uuid::new_v4()).collect::<Vec<_>>();
|
||||
let deployment_id = Uuid::new_v4();
|
||||
let mut endpoints = Vec::new();
|
||||
for slot in 0..3 {
|
||||
let path = temp.path().join(format!("target-{slot}"));
|
||||
let metadata_path = path.join(crate::heal::RUSTFS_META_BUCKET);
|
||||
std::fs::create_dir_all(&metadata_path).expect("scanner local target metadata directory");
|
||||
if !unformatted_siblings || slot == 0 {
|
||||
let format = serde_json::json!({
|
||||
"version": "1",
|
||||
"format": "xl",
|
||||
"id": deployment_id,
|
||||
"xl": {
|
||||
"version": "3",
|
||||
"this": disk_ids[slot],
|
||||
"sets": [disk_ids],
|
||||
"distributionAlgo": "SIPMOD+PARITY"
|
||||
}
|
||||
});
|
||||
std::fs::write(
|
||||
metadata_path.join("format.json"),
|
||||
serde_json::to_vec(&format).expect("scanner format fixture serialization"),
|
||||
)
|
||||
.expect("scanner formatted target fixture");
|
||||
}
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(path.to_str().expect("scanner target path is UTF-8")).expect("scanner local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(slot);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
let local_targets = endpoints.iter().map(ToString::to_string).collect::<Vec<_>>();
|
||||
init_local_disks(EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: 3,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "replacement-scanner-recovery-test".into(),
|
||||
platform: "test".into(),
|
||||
}]))
|
||||
.await
|
||||
.expect("scanner local registry initialization");
|
||||
|
||||
let survivor = make_manager_resume_disk(&temp, "survivor").await;
|
||||
let local_disks = crate::heal::local_disk_map_read().await;
|
||||
assert_eq!(local_disks.len(), 3);
|
||||
assert!(
|
||||
!local_disks.contains_key(&survivor.endpoint().to_string()),
|
||||
"the durable anchor must be discoverable only through the set inventory"
|
||||
);
|
||||
for disk in local_disks.values().flatten() {
|
||||
let is_owner = disk.endpoint().to_string() == local_targets[0];
|
||||
match disk.get_disk_id().await {
|
||||
Ok(Some(id)) => assert!(!id.is_nil()),
|
||||
Err(DiskError::UnformattedDisk) => assert!(unformatted_siblings && !is_owner),
|
||||
result => panic!("unexpected fixture disk identity: {result:?}"),
|
||||
}
|
||||
assert_eq!(disk.runtime_state().as_str(), "online");
|
||||
}
|
||||
drop(local_disks);
|
||||
|
||||
let identity = ReplacementTargetIdentity {
|
||||
endpoint: local_targets[0].clone(),
|
||||
canonical_path: local_targets[0].clone(),
|
||||
physical_device_ids: vec!["replacement-target-0".into()],
|
||||
filesystem_identity: "replacement-target-0-incarnation".into(),
|
||||
};
|
||||
let task_id = ResumeUtils::generate_task_id();
|
||||
let owner = ResumeManager::new_replacement_intent(
|
||||
survivor.clone(),
|
||||
task_id.clone(),
|
||||
SET.into(),
|
||||
vec!["seeded-bucket".into()],
|
||||
vec![local_targets[0].clone()],
|
||||
vec![identity.clone()],
|
||||
)
|
||||
.await
|
||||
.expect("durable remote-survivor replacement intent");
|
||||
owner
|
||||
.mark_replacement_rebuilding(vec![identity.clone()])
|
||||
.await
|
||||
.expect("durable replacement rebuild phase");
|
||||
owner
|
||||
.record_replacement_failure(&Error::Storage(EcstoreError::FileNotFound), 1)
|
||||
.await
|
||||
.expect("durable in-budget FileNotFound failure");
|
||||
let storage = Arc::new(ScannerStorage {
|
||||
survivor,
|
||||
identity,
|
||||
inventory_calls: AtomicUsize::new(0),
|
||||
});
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
storage.clone(),
|
||||
Some(HealConfig {
|
||||
heal_interval: SCAN_INTERVAL,
|
||||
..HealConfig::default()
|
||||
}),
|
||||
);
|
||||
Self {
|
||||
_temp: temp,
|
||||
manager,
|
||||
storage,
|
||||
owner,
|
||||
task_id,
|
||||
local_targets,
|
||||
}
|
||||
}
|
||||
|
||||
async fn queued_request(&self) -> HealRequest {
|
||||
// Keep the paused clock from advancing while filesystem workers
|
||||
// finish. A wall-clock deadline also bounds a broken admission path.
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
let queue = self.manager.heal_queue.lock().await;
|
||||
if let Some(item) = queue.heap.peek() {
|
||||
assert_eq!(queue.len(), 1, "one set must have exactly one admitted owner");
|
||||
return item.request.clone();
|
||||
}
|
||||
drop(queue);
|
||||
assert!(std::time::Instant::now() < deadline, "scanner tick did not admit replacement recovery");
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_original_owner_is_queued(&self) {
|
||||
let request = self.queued_request().await;
|
||||
assert_eq!(request.id, self.task_id, "scanner must replay the durable UUID");
|
||||
assert_eq!(request.heal_endpoints, vec![self.local_targets[0].clone()]);
|
||||
assert_eq!(request.source, HealRequestSource::AutoHeal);
|
||||
assert!(matches!(
|
||||
request.heal_type,
|
||||
HealType::ErasureSet { set_disk_id, buckets }
|
||||
if set_disk_id == SET && buckets == ["seeded-bucket"]
|
||||
));
|
||||
assert_eq!(
|
||||
self.manager
|
||||
.replacement_recovery_anchors
|
||||
.lock()
|
||||
.expect("scanner anchor registry")
|
||||
.get(&self.task_id),
|
||||
Some(&self.storage.survivor.endpoint().to_string())
|
||||
);
|
||||
assert!(!self.manager.replacement_recovery_set_is_blocked(SET));
|
||||
assert_eq!(
|
||||
self.storage.inventory_calls.load(Ordering::SeqCst),
|
||||
1,
|
||||
"three local slots in one set must share one inventory lookup per tick"
|
||||
);
|
||||
let state = self.owner.get_state().await;
|
||||
assert_eq!(state.retry_count, 1, "discovery must not reset the durable retry budget");
|
||||
assert_eq!(state.replacement_targets, request.heal_endpoints);
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_without_advancing_clock<T>(future: impl Future<Output = T>) -> T {
|
||||
tokio::pin!(future);
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
result = &mut future => return result,
|
||||
_ = tokio::task::yield_now() => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial_test::serial(replacement_auto_scan)]
|
||||
async fn scanner_tick_recovers_remote_survivor_after_all_local_targets_are_formatted() {
|
||||
let fixture = ScannerFixture::new(false).await;
|
||||
fixture
|
||||
.manager
|
||||
.start_auto_disk_scanner()
|
||||
.await
|
||||
.expect("start only the periodic replacement scanner");
|
||||
fixture.assert_original_owner_is_queued().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
#[serial_test::serial(replacement_auto_scan)]
|
||||
async fn scanner_tick_finishes_existing_owner_before_admitting_new_unformatted_targets() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], async {
|
||||
let fixture = ScannerFixture::new(true).await;
|
||||
fixture
|
||||
.manager
|
||||
.start_auto_disk_scanner()
|
||||
.await
|
||||
.expect("start only the periodic replacement scanner");
|
||||
fixture.assert_original_owner_is_queued().await;
|
||||
|
||||
// The real task's verified completion publishes the proof and
|
||||
// removes its live intent before sibling admission is allowed.
|
||||
let owner_removed = finish_without_advancing_clock(async {
|
||||
fixture
|
||||
.owner
|
||||
.mark_replacement_completed_and_verified()
|
||||
.await
|
||||
.expect("durable owner completion proof");
|
||||
fixture
|
||||
.owner
|
||||
.mark_replacement_cleanup_pending()
|
||||
.await
|
||||
.expect("durable owner cleanup phase");
|
||||
fixture
|
||||
.owner
|
||||
.cleanup()
|
||||
.await
|
||||
.expect("completed owner releases its durable intent");
|
||||
!ResumeManager::has_replacement_intent(&fixture.storage.survivor, &fixture.task_id).await
|
||||
})
|
||||
.await;
|
||||
assert!(owner_removed, "completed ownership must be removed before the next generation");
|
||||
let finished = fixture
|
||||
.manager
|
||||
.heal_queue
|
||||
.lock()
|
||||
.await
|
||||
.pop_next()
|
||||
.expect("remove the completed queued owner");
|
||||
assert_eq!(finished.id, fixture.task_id);
|
||||
tokio::time::advance(SCAN_INTERVAL).await;
|
||||
|
||||
let next = fixture.queued_request().await;
|
||||
assert_ne!(next.id, fixture.task_id, "the follow-up replacement gets a fresh generation");
|
||||
assert_eq!(next.heal_endpoints, fixture.local_targets[1..]);
|
||||
assert_eq!(next.source, HealRequestSource::AutoHeal);
|
||||
assert!(!fixture.manager.replacement_recovery_set_is_blocked(SET));
|
||||
assert_eq!(fixture.storage.inventory_calls.load(Ordering::SeqCst), 2);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovery_disk_index_keeps_first_snapshot_and_sorted_unique_endpoints() {
|
||||
let temp = TempDir::new().expect("recovery disk index fixture");
|
||||
let local = make_manager_resume_disk(&temp, "a-local").await;
|
||||
let duplicate = make_manager_resume_disk(&temp, "a-local").await;
|
||||
let survivor = make_manager_resume_disk(&temp, "z-survivor").await;
|
||||
assert!(!Arc::ptr_eq(&local, &duplicate), "duplicate must be a different disk snapshot");
|
||||
let local_endpoint = local.endpoint().to_string();
|
||||
let survivor_endpoint = survivor.endpoint().to_string();
|
||||
let mut inventory = std::collections::BTreeMap::new();
|
||||
super::super::auto_scan::extend_replacement_recovery_disks(&mut inventory, [local.clone()]);
|
||||
super::super::auto_scan::extend_replacement_recovery_disks(&mut inventory, [survivor.clone(), duplicate, survivor]);
|
||||
assert_eq!(inventory.len(), 2, "each endpoint is discovered once");
|
||||
assert!(
|
||||
Arc::ptr_eq(inventory.get(&local_endpoint).expect("local endpoint remains indexed"), &local),
|
||||
"later set inventories must not replace the local snapshot"
|
||||
);
|
||||
assert_eq!(
|
||||
inventory.keys().collect::<Vec<_>>(),
|
||||
vec![&local_endpoint, &survivor_endpoint],
|
||||
"discovery order remains lexical regardless of insertion order"
|
||||
);
|
||||
}
|
||||
@@ -1844,7 +1844,7 @@ async fn root_recovery_queued_non_root_admin_owner_is_not_priority_displaced() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_legacy_schema_replays_as_cluster() {
|
||||
async fn root_recovery_legacy_cluster_requires_a_fresh_generation_bound_task() {
|
||||
#[derive(serde::Serialize)]
|
||||
struct LegacyRootHealIntent<'a> {
|
||||
schema: u32,
|
||||
@@ -1873,10 +1873,113 @@ async fn root_recovery_legacy_schema_replays_as_cluster() {
|
||||
|
||||
let restarted = recovery_manager(vec![disk]);
|
||||
restarted.replay_root_heals().await.expect("replay legacy root record");
|
||||
let queued = restarted.heal_queue.lock().await.requests().cloned().collect::<Vec<_>>();
|
||||
assert_eq!(queued.len(), 1);
|
||||
assert_eq!(queued[0].id, request.id);
|
||||
assert_eq!(queued[0].heal_type, HealType::Cluster);
|
||||
assert!(
|
||||
restarted.heal_queue.lock().await.requests().next().is_none(),
|
||||
"unfenced legacy work must not bind to a later bucket generation"
|
||||
);
|
||||
assert!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.inventory()
|
||||
.await
|
||||
.expect("quarantined legacy task")
|
||||
.quarantined_task_ids
|
||||
.contains(&request.id)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_cluster_scope_preserves_generations_and_completed_prefix_on_handoff() {
|
||||
use crate::heal::manager::root_recovery::AdminClusterScope;
|
||||
use std::collections::BTreeMap;
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk.clone()]);
|
||||
let request = root_request();
|
||||
manager.root_recovery.persist(&request).await.expect("admit cluster task");
|
||||
let task = HealTask::from_request(request.clone(), Arc::new(MockStorage));
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
let mut scope = AdminClusterScope {
|
||||
buckets: BTreeMap::from([
|
||||
("alpha-bucket".to_owned(), Uuid::from_u128(42)),
|
||||
("bravo-bucket".to_owned(), Uuid::from_u128(43)),
|
||||
]),
|
||||
next_bucket_index: 0,
|
||||
outcome,
|
||||
progress: HealProgress::new(),
|
||||
};
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("freeze bucket generations before mutation");
|
||||
scope.next_bucket_index = 1;
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("checkpoint completed empty bucket");
|
||||
let retry = task.retry_request();
|
||||
manager
|
||||
.root_recovery
|
||||
.persist(&retry)
|
||||
.await
|
||||
.expect("shutdown/retry handoff retains scope");
|
||||
let mut changed_options = retry.clone();
|
||||
changed_options.options.recursive = !changed_options.options.recursive;
|
||||
assert!(
|
||||
manager.root_recovery.persist(&changed_options).await.is_err(),
|
||||
"handoff cannot reinterpret completed work"
|
||||
);
|
||||
drop(manager);
|
||||
let restarted = recovery_manager(vec![disk]);
|
||||
let replay = HealTask::from_request(retry, Arc::new(MockStorage));
|
||||
assert_eq!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.cluster_scope(&replay)
|
||||
.await
|
||||
.expect("read frozen scope"),
|
||||
Some(scope.clone())
|
||||
);
|
||||
let mut changed = scope.clone();
|
||||
changed.buckets.insert("alpha-bucket".to_owned(), Uuid::from_u128(44));
|
||||
assert!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&replay, &changed)
|
||||
.await
|
||||
.is_err(),
|
||||
"never rebind a saved generation"
|
||||
);
|
||||
changed = scope.clone();
|
||||
changed.next_bucket_index = 0;
|
||||
assert!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&replay, &changed)
|
||||
.await
|
||||
.is_err(),
|
||||
"never rewind a committed prefix"
|
||||
);
|
||||
changed = scope.clone();
|
||||
changed.buckets.insert("../invalid".to_owned(), Uuid::from_u128(45));
|
||||
assert!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&replay, &changed)
|
||||
.await
|
||||
.is_err(),
|
||||
"reject decoded path aliases"
|
||||
);
|
||||
assert_eq!(
|
||||
restarted
|
||||
.root_recovery
|
||||
.cluster_scope(&replay)
|
||||
.await
|
||||
.expect("failed updates preserve scope"),
|
||||
Some(scope)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1899,6 +2002,155 @@ async fn root_recovery_rejected_admin_start_does_not_persist() {
|
||||
assert_eq!(restarted.get_queue_length().await, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_cluster_replay_rejects_successor_and_preserves_completed_scope() {
|
||||
use crate::heal::manager::root_recovery::AdminClusterScope;
|
||||
use crate::heal::outcome::{HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectOutcome};
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk]);
|
||||
let request = root_request();
|
||||
manager.root_recovery.persist(&request).await.expect("admit cluster task");
|
||||
let task = HealTask::from_request(request.clone(), Arc::new(MockStorage));
|
||||
let bucket = format!("pinned-{}", Uuid::new_v4().simple());
|
||||
let expected = Uuid::from_u128(42);
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
let mut scope = AdminClusterScope {
|
||||
buckets: std::collections::BTreeMap::from([(bucket.clone(), expected)]),
|
||||
next_bucket_index: 0,
|
||||
outcome,
|
||||
progress: HealProgress::new(),
|
||||
};
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("freeze generation");
|
||||
bucket_incarnations()
|
||||
.lock()
|
||||
.expect("generation fixture")
|
||||
.insert(bucket.clone(), Some(Uuid::from_u128(99)));
|
||||
let replay =
|
||||
HealTask::from_request(request.clone(), Arc::new(MockStorage)).with_admin_recovery(manager.root_recovery.clone());
|
||||
assert!(
|
||||
matches!(replay.execute().await, Err(Error::StaleBucketIncarnation { .. })),
|
||||
"replay must reject a successor before traversal"
|
||||
);
|
||||
assert_eq!(replay.get_outcome().await.counters.processed, 0);
|
||||
bucket_incarnations()
|
||||
.lock()
|
||||
.expect("generation fixture")
|
||||
.insert(bucket.clone(), Some(expected));
|
||||
scope.next_bucket_index = 1;
|
||||
scope.outcome.record(HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: bucket.clone(),
|
||||
object: "completed-object".to_owned(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: Some(expected),
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
disposition: HealObjectDisposition::VerifiedHealthy,
|
||||
detail: None,
|
||||
});
|
||||
scope.progress.objects_scanned = 1;
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("completed bucket with retained outcome");
|
||||
let replay = HealTask::from_request(request, Arc::new(MockStorage)).with_admin_recovery(manager.root_recovery.clone());
|
||||
replay
|
||||
.execute()
|
||||
.await
|
||||
.expect("completed frozen scope needs no rediscovery or repair");
|
||||
assert_eq!(replay.get_outcome().await.coverage, crate::heal::outcome::HealTraversalCoverage::Complete);
|
||||
assert_eq!(replay.get_outcome().await.counters.processed, 1);
|
||||
assert_eq!(replay.get_progress().await.objects_scanned, 1);
|
||||
bucket_incarnations().lock().expect("generation fixture").remove(&bucket);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_cluster_inventory_exceeds_legacy_limit_without_losing_bindings() {
|
||||
use crate::heal::manager::root_recovery::AdminClusterScope;
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk.clone()]);
|
||||
let request = root_request();
|
||||
manager.root_recovery.persist(&request).await.expect("admit cluster task");
|
||||
let task = HealTask::from_request(request.clone(), Arc::new(MockStorage));
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
let scope = AdminClusterScope {
|
||||
buckets: (0..1500)
|
||||
.map(|index| (format!("bucket-{index:05}"), Uuid::from_u128(42)))
|
||||
.collect(),
|
||||
next_bucket_index: 0,
|
||||
outcome,
|
||||
progress: HealProgress::new(),
|
||||
};
|
||||
assert!(scope.checkpoint_interval() > 1, "large inventories must batch durable rewrites");
|
||||
manager
|
||||
.root_recovery
|
||||
.checkpoint_cluster_scope(&task, &scope)
|
||||
.await
|
||||
.expect("persist large frozen inventory");
|
||||
let path = format!("root-heal-{}.json", request.id);
|
||||
let bytes = disk.read_all(RUSTFS_META_BUCKET, &path).await.expect("scope bytes");
|
||||
assert!(bytes.len() > MAX_ROOT_RECOVERY_BYTES, "fixture must cross legacy size limit");
|
||||
assert_eq!(
|
||||
manager
|
||||
.root_recovery
|
||||
.cluster_scope(&task)
|
||||
.await
|
||||
.expect("all bindings survive"),
|
||||
Some(scope)
|
||||
);
|
||||
let json = String::from_utf8(bytes.to_vec()).expect("UTF-8 scope");
|
||||
let binding = format!("\"bucket-00000\":\"{}\"", Uuid::from_u128(42));
|
||||
let duplicate = json.replacen(&binding, &format!("{binding},{binding}"), 1);
|
||||
assert_ne!(duplicate, json, "inject a duplicate map key");
|
||||
disk.write_all(RUSTFS_META_BUCKET, &path, duplicate.into_bytes().into())
|
||||
.await
|
||||
.expect("inject corrupt duplicate binding");
|
||||
assert!(
|
||||
manager.root_recovery.cluster_scope(&task).await.is_err(),
|
||||
"duplicate identity bindings fail closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_cluster_rejects_inventory_without_checkpoint_headroom() {
|
||||
use crate::heal::manager::root_recovery::AdminClusterScope;
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk]);
|
||||
let request = root_request();
|
||||
manager.root_recovery.persist(&request).await.expect("admit cluster task");
|
||||
let task = HealTask::from_request(request, Arc::new(MockStorage));
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.start();
|
||||
let scope = AdminClusterScope {
|
||||
// This inventory fits the 16 MiB record limit but leaves too little
|
||||
// space for future object outcomes. Reject it before authorizing repair.
|
||||
buckets: (0..152_000)
|
||||
.map(|index| (format!("bucket-{index:056}"), Uuid::from_u128(42)))
|
||||
.collect(),
|
||||
next_bucket_index: 0,
|
||||
outcome,
|
||||
progress: HealProgress::new(),
|
||||
};
|
||||
assert!(manager.root_recovery.checkpoint_cluster_scope(&task, &scope).await.is_err());
|
||||
assert!(
|
||||
manager
|
||||
.root_recovery
|
||||
.cluster_scope(&task)
|
||||
.await
|
||||
.expect("original admission survives")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_queued_owner_is_not_priority_displaced() {
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
@@ -2127,7 +2379,16 @@ async fn root_recovery_force_start_replaces_fresh_queued_and_retrying_admin_root
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay() {
|
||||
for kind in ["truncated", "schema", "identity", "option", "no_lock", "oversized"] {
|
||||
use crate::heal::manager::root_recovery::MAX_CLUSTER_RECOVERY_BYTES;
|
||||
for kind in [
|
||||
"truncated",
|
||||
"schema",
|
||||
"identity",
|
||||
"option",
|
||||
"no_lock",
|
||||
"unscoped_oversized",
|
||||
"oversized",
|
||||
] {
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk.clone()]);
|
||||
let valid = root_request();
|
||||
@@ -2142,7 +2403,7 @@ async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay()
|
||||
let original = disk.read_all(RUSTFS_META_BUCKET, &path).await.expect("read root record");
|
||||
let mut value: serde_json::Value = serde_json::from_slice(&original).expect("record JSON");
|
||||
match kind {
|
||||
"schema" => value["schema"] = 4.into(),
|
||||
"schema" => value["schema"] = u32::MAX.into(),
|
||||
"identity" => value["task_id"] = valid.id.clone().into(),
|
||||
"option" => value["options"]["future_delete_mode"] = true.into(),
|
||||
"no_lock" => value["options"]["no_lock"] = true.into(),
|
||||
@@ -2150,7 +2411,12 @@ async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay()
|
||||
}
|
||||
let bytes = match kind {
|
||||
"truncated" => b"{".to_vec(),
|
||||
"oversized" => vec![b' '; MAX_ROOT_RECOVERY_BYTES + 1],
|
||||
"oversized" => vec![b' '; MAX_CLUSTER_RECOVERY_BYTES + 1],
|
||||
"unscoped_oversized" => {
|
||||
let mut bytes = serde_json::to_vec(&value).expect("valid unscoped record");
|
||||
bytes.resize(MAX_ROOT_RECOVERY_BYTES + 1, b' ');
|
||||
bytes
|
||||
}
|
||||
_ => serde_json::to_vec(&value).expect("modified record"),
|
||||
};
|
||||
disk.write_all(RUSTFS_META_BUCKET, &path, bytes.clone().into())
|
||||
@@ -2173,7 +2439,11 @@ async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay()
|
||||
let marker: serde_json::Value = serde_json::from_slice(&marker).expect("quarantine marker JSON");
|
||||
assert_eq!(marker["task_id"], invalid.id, "kind={kind}");
|
||||
assert_eq!(marker["record_kind"], "intent", "kind={kind}");
|
||||
assert_eq!(marker["source_prefix_len"], bytes.len().min(MAX_ROOT_RECOVERY_BYTES + 1), "kind={kind}");
|
||||
assert_eq!(
|
||||
marker["source_prefix_len"],
|
||||
bytes.len().min(MAX_CLUSTER_RECOVERY_BYTES + 1),
|
||||
"kind={kind}"
|
||||
);
|
||||
assert_eq!(marker["source_oversized"], kind == "oversized", "kind={kind}");
|
||||
|
||||
let mut reused = invalid.clone();
|
||||
@@ -2195,6 +2465,51 @@ async fn root_recovery_invalid_records_are_quarantined_without_blocking_replay()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_legacy_oversized_quarantine_survives_larger_intent_limit() {
|
||||
use sha2::{Digest, Sha256};
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk.clone()]);
|
||||
let request = root_request();
|
||||
let path = format!("root-heal-{}.json", request.id);
|
||||
let mut bytes = vec![b' '; MAX_ROOT_RECOVERY_BYTES + 100];
|
||||
disk.write_all(RUSTFS_META_BUCKET, &path, bytes.clone().into())
|
||||
.await
|
||||
.expect("old oversized intent");
|
||||
let marker = serde_json::json!({
|
||||
"schema": 1,
|
||||
"task_id": request.id,
|
||||
"record_kind": "intent",
|
||||
"source_prefix_len": MAX_ROOT_RECOVERY_BYTES + 1,
|
||||
"source_prefix_sha256": Sha256::digest(&bytes[..MAX_ROOT_RECOVERY_BYTES + 1]).to_vec(),
|
||||
"source_oversized": true,
|
||||
"reason": "legacy size limit",
|
||||
"quarantined_at": SystemTime::now(),
|
||||
});
|
||||
disk.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
&format!("quarantined-root-heal-intent-{}.json", request.id),
|
||||
serde_json::to_vec(&marker).unwrap().into(),
|
||||
)
|
||||
.await
|
||||
.expect("old quarantine marker");
|
||||
let inventory = manager
|
||||
.root_recovery
|
||||
.inventory()
|
||||
.await
|
||||
.expect("upgrade preserves old quarantine");
|
||||
assert!(inventory.requests.is_empty());
|
||||
assert!(inventory.quarantined_task_ids.contains(&request.id));
|
||||
bytes[0] = b'{';
|
||||
disk.write_all(RUSTFS_META_BUCKET, &path, bytes.into())
|
||||
.await
|
||||
.expect("alter retained prefix");
|
||||
assert!(
|
||||
manager.root_recovery.inventory().await.is_err(),
|
||||
"a changed legacy source still fails closed"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn root_recovery_quarantine_source_change_fails_closed() {
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
|
||||
@@ -80,6 +80,13 @@ impl HealStorageAPI for RunningStorage {
|
||||
..Default::default()
|
||||
}))
|
||||
}
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
_selector: Option<crate::heal::storage_api::storage::DiskSetSelector>,
|
||||
) -> Result<Vec<BucketInfo>> {
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
@@ -491,6 +491,10 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
/// Get all buckets
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>>;
|
||||
|
||||
/// Return only a complete bucket inventory for an administrator task.
|
||||
/// Incomplete discovery must fail before traversal or scope checkpointing.
|
||||
async fn list_buckets_for_admin_heal(&self, selector: Option<DiskSetSelector>) -> Result<Vec<BucketInfo>>;
|
||||
|
||||
/// Check object exists
|
||||
async fn object_exists(&self, bucket: &str, object: &str) -> Result<bool>;
|
||||
|
||||
@@ -1335,6 +1339,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_buckets_for_admin_heal(&self, selector: Option<DiskSetSelector>) -> Result<Vec<BucketInfo>> {
|
||||
let listing = self.ecstore.list_buckets_for_admin_heal(selector).await?;
|
||||
if !listing.topology_complete {
|
||||
return Err(Error::BucketDiscoveryIncomplete);
|
||||
}
|
||||
Ok(listing.buckets)
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) use rustfs_ecstore::api::capacity::is_reserved_or_invalid_bucket as ecstore_is_reserved_or_invalid_bucket;
|
||||
pub(crate) use rustfs_ecstore::api::data_usage::{
|
||||
DATA_USAGE_CACHE_NAME as ECSTORE_DATA_USAGE_CACHE_NAME,
|
||||
load_admin_data_usage_from_backend_cached as ecstore_load_admin_data_usage_from_backend_cached,
|
||||
@@ -26,7 +27,13 @@ pub(crate) use rustfs_ecstore::api::disk::{
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::disk::{DiskOption as EcstoreDiskOption, new_disk as ecstore_new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::error::{Error as EcstoreErrorType, StorageError as EcstoreStorageError};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::layout::{
|
||||
EndpointServerPools as EcstoreEndpointServerPools, Endpoints as EcstoreEndpoints, PoolEndpoints as EcstorePoolEndpoints,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::runtime::local_disk_map_read as ecstore_local_disk_map_read;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks as ecstore_init_local_disks;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore as EcstoreStore, HealLifecycleExpiryContext as EcstoreHealLifecycleExpiryContext,
|
||||
HealObjectStorageResult as EcstoreHealObjectStorageResult, POOL_META_NAME as ECSTORE_POOL_META_NAME,
|
||||
@@ -40,8 +47,8 @@ pub(crate) mod owner {
|
||||
ECSTORE_BUCKET_META_PREFIX, ECSTORE_DATA_USAGE_CACHE_NAME, ECSTORE_HEALING_MARKER_PATH, ECSTORE_POOL_META_NAME,
|
||||
ECSTORE_RUSTFS_META_BUCKET, EcstoreConditionalFileUpdate, EcstoreDeleteOptions, EcstoreDiskAPI, EcstoreDiskBytes,
|
||||
EcstoreDiskError, EcstoreDiskResult, EcstoreDiskStore, EcstoreEndpoint, EcstoreErrorType,
|
||||
EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore, ecstore_load_admin_data_usage_from_backend_cached,
|
||||
ecstore_local_disk_map_read,
|
||||
EcstoreHealLifecycleExpiryContext, EcstoreStorageError, EcstoreStore, ecstore_is_reserved_or_invalid_bucket,
|
||||
ecstore_load_admin_data_usage_from_backend_cached, ecstore_local_disk_map_read,
|
||||
};
|
||||
|
||||
pub(crate) use super::{EcstoreDiskOption, ecstore_new_disk};
|
||||
|
||||
@@ -132,6 +132,19 @@ fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Optio
|
||||
}
|
||||
|
||||
impl HealTask {
|
||||
async fn recover_bucket_delete_transactions(&self, bucket: &str, opts: &HealOpts) -> Result<()> {
|
||||
if let Some(result) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.recover_delete_transactions(bucket, None, self.bucket_incarnation_id, opts),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
|
||||
self.pace_mainline().await?;
|
||||
if self.source == HealRequestSource::Admin && matches!(self.heal_type, HealType::Bucket { .. }) {
|
||||
@@ -169,8 +182,10 @@ impl HealTask {
|
||||
"Heal bucket stage entered"
|
||||
);
|
||||
self.check_control_flags().await?;
|
||||
let bucket_exists = self.await_with_control(self.storage.get_bucket_info(bucket)).await?.is_some();
|
||||
if !bucket_exists {
|
||||
// Bound administrator repair validates existence and generation under the
|
||||
// lifecycle fence. Ordinary read quorum can reject recoverable directory loss.
|
||||
let bound_admin_repair = self.source == HealRequestSource::Admin && self.bucket_incarnation_id.is_some();
|
||||
if !bound_admin_repair && self.await_with_control(self.storage.get_bucket_info(bucket)).await?.is_none() {
|
||||
warn!(
|
||||
target: "rustfs::heal::task",
|
||||
event = EVENT_HEAL_BUCKET_RESULT,
|
||||
@@ -224,18 +239,11 @@ impl HealTask {
|
||||
remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted,
|
||||
..heal_opts
|
||||
};
|
||||
if let Some(result) = self
|
||||
.await_with_control(self.storage.recover_delete_transactions(
|
||||
bucket,
|
||||
None,
|
||||
self.bucket_incarnation_id,
|
||||
&recovery_opts,
|
||||
))
|
||||
.await?
|
||||
{
|
||||
self.record_result_item(result).await;
|
||||
// Removal must settle survivor journals before deleting an empty
|
||||
// replica's publication directory; repair must establish it first.
|
||||
if heal_opts.remove {
|
||||
self.recover_bucket_delete_transactions(bucket, &recovery_opts).await?;
|
||||
}
|
||||
|
||||
let heal_result = match self.bucket_incarnation_id {
|
||||
Some(expected) => {
|
||||
self.await_with_control(self.storage.heal_bucket_at_incarnation(bucket, expected, &heal_opts))
|
||||
@@ -261,6 +269,10 @@ impl HealTask {
|
||||
);
|
||||
self.record_result_item(result).await;
|
||||
|
||||
if !heal_opts.remove {
|
||||
self.recover_bucket_delete_transactions(bucket, &recovery_opts).await?;
|
||||
}
|
||||
|
||||
if self.options.recursive {
|
||||
self.heal_bucket_objects(bucket, "").await?;
|
||||
}
|
||||
@@ -308,17 +320,76 @@ impl HealTask {
|
||||
"Heal cluster started"
|
||||
);
|
||||
|
||||
let bucket_infos = self.await_with_control(self.storage.list_buckets()).await?;
|
||||
let mut cluster_scope = if self.source == HealRequestSource::Admin && !self.options.dry_run {
|
||||
let saved = if let Some(recovery) = &self.admin_recovery {
|
||||
recovery.cluster_scope(self).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let scope = if let Some(scope) = saved {
|
||||
scope
|
||||
} else {
|
||||
let mut buckets = std::collections::BTreeMap::new();
|
||||
for bucket in self
|
||||
.await_with_control(self.storage.list_buckets_for_admin_heal(None))
|
||||
.await?
|
||||
{
|
||||
let incarnation = self
|
||||
.await_with_control(self.storage.admit_bucket_incarnation(&bucket.name))
|
||||
.await?;
|
||||
buckets.insert(bucket.name, incarnation);
|
||||
}
|
||||
let scope = super::super::manager::root_recovery::AdminClusterScope {
|
||||
buckets,
|
||||
next_bucket_index: 0,
|
||||
outcome: self.get_outcome().await,
|
||||
progress: self.progress.read().await.clone(),
|
||||
};
|
||||
if let Some(recovery) = &self.admin_recovery {
|
||||
recovery.checkpoint_cluster_scope(self, &scope).await?;
|
||||
}
|
||||
scope
|
||||
};
|
||||
// Validate the frozen generations on every replay, including completed buckets.
|
||||
for (bucket, incarnation) in &scope.buckets {
|
||||
self.await_with_control(self.storage.validate_bucket_incarnation(bucket, Some(*incarnation)))
|
||||
.await?;
|
||||
}
|
||||
self.restore_outcome(scope.outcome.clone()).await;
|
||||
*self.progress.write().await = scope.progress.clone();
|
||||
Some(scope)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let buckets = if let Some(scope) = &cluster_scope {
|
||||
scope
|
||||
.buckets
|
||||
.iter()
|
||||
.map(|(name, incarnation)| (name.clone(), Some(*incarnation)))
|
||||
.collect::<Vec<_>>()
|
||||
} else {
|
||||
let listed = if self.source == HealRequestSource::Admin {
|
||||
self.await_with_control(self.storage.list_buckets_for_admin_heal(None))
|
||||
.await?
|
||||
} else {
|
||||
self.await_with_control(self.storage.list_buckets()).await?
|
||||
};
|
||||
listed.into_iter().map(|bucket| (bucket.name, None)).collect()
|
||||
};
|
||||
let mut failed = 0_u64;
|
||||
let mut retryable = 0_u64;
|
||||
let mut permanent = 0_u64;
|
||||
let mut first_object = None;
|
||||
let mut first_error = None;
|
||||
for bucket_info in bucket_infos {
|
||||
let start = cluster_scope.as_ref().map_or(0, |scope| scope.next_bucket_index);
|
||||
let mut checkpointed = start;
|
||||
for (index, (bucket, incarnation)) in buckets.into_iter().enumerate().skip(start) {
|
||||
self.check_control_flags().await?;
|
||||
let mut bucket_task = self.clone();
|
||||
bucket_task.bucket_incarnation_id = incarnation;
|
||||
let mut retry_attempt = 0_u32;
|
||||
loop {
|
||||
match self.heal_bucket(&bucket_info.name).await {
|
||||
match bucket_task.heal_bucket(&bucket).await {
|
||||
Ok(()) => break,
|
||||
Err(Error::TaskCancelled) => return Err(Error::TaskCancelled),
|
||||
Err(Error::TaskTimeout) => return Err(Error::TaskTimeout),
|
||||
@@ -347,15 +418,37 @@ impl HealTask {
|
||||
} else {
|
||||
permanent = permanent.saturating_add(1);
|
||||
}
|
||||
first_object.get_or_insert(bucket_info.name.clone());
|
||||
first_object.get_or_insert(bucket.clone());
|
||||
first_error.get_or_insert_with(|| err.to_string());
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Retain only the successful contiguous prefix. A later retry replays
|
||||
// any work after a failure without inheriting its transient counters.
|
||||
if failed == 0
|
||||
&& let Some(scope) = &mut cluster_scope
|
||||
{
|
||||
scope.next_bucket_index = index.saturating_add(1);
|
||||
scope.outcome = self.get_outcome().await;
|
||||
scope.progress = self.progress.read().await.clone();
|
||||
if (scope.next_bucket_index.saturating_sub(checkpointed) >= scope.checkpoint_interval()
|
||||
|| scope.next_bucket_index == scope.buckets.len())
|
||||
&& let Some(recovery) = &self.admin_recovery
|
||||
{
|
||||
recovery.checkpoint_cluster_scope(self, scope).await?;
|
||||
checkpointed = scope.next_bucket_index;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if failed > 0 {
|
||||
if let Some(scope) = &cluster_scope
|
||||
&& scope.next_bucket_index > checkpointed
|
||||
&& let Some(recovery) = &self.admin_recovery
|
||||
{
|
||||
recovery.checkpoint_cluster_scope(self, scope).await?;
|
||||
}
|
||||
let failure = BatchHealFailure {
|
||||
scope: "cluster".to_string(),
|
||||
failed,
|
||||
|
||||
@@ -460,21 +460,6 @@ impl HealTask {
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
let recovery_opts = HealOpts {
|
||||
remove: self.source == HealRequestSource::Admin && self.options.remove_corrupted,
|
||||
..bucket_heal_opts
|
||||
};
|
||||
// Include metadata-less transactions for both startup and admin heals.
|
||||
for bucket in &buckets {
|
||||
self.check_control_flags().await?;
|
||||
if let Some(result) = self
|
||||
.await_with_control(self.storage.recover_delete_transactions(bucket, None, None, &recovery_opts))
|
||||
.await?
|
||||
{
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
}
|
||||
|
||||
for bucket in buckets.iter().filter(|_| self.source != HealRequestSource::Admin) {
|
||||
// Check control flags before starting each bucket heal
|
||||
self.check_control_flags().await?;
|
||||
@@ -505,6 +490,16 @@ impl HealTask {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
// Adopt delete proofs only after their publication bucket exists.
|
||||
if let Some(result) = self
|
||||
.await_with_control(
|
||||
self.storage
|
||||
.recover_delete_transactions(bucket, None, None, &bucket_heal_opts),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
self.record_result_item(result).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Create erasure set healer with resume support
|
||||
|
||||
@@ -24,6 +24,59 @@ mod canonical_outcome {
|
||||
use super::*;
|
||||
use crate::heal::outcome::{HealExecutionOutcome, HealTraversalCoverage};
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_bucket_discovery_incomplete_cannot_finish_an_empty_cluster() {
|
||||
for dry_run in [false, true] {
|
||||
let storage = Arc::new(MockStorage {
|
||||
listed_buckets: Mutex::new(Some(Vec::new())),
|
||||
bucket_discovery_incomplete: true,
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Cluster,
|
||||
HealOptions {
|
||||
dry_run,
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
assert!(matches!(task.execute().await, Err(Error::BucketDiscoveryIncomplete)));
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Aborted(HealAbortReason::Untraversable));
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
assert_eq!(outcome.counters.processed, 0);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert!(storage.bucket_heal_calls.lock().expect("bucket calls").is_empty());
|
||||
assert!(storage.heal_object_calls.lock().expect("object calls").is_empty());
|
||||
let wire = serde_json::to_value(&outcome).expect("outcome serializes");
|
||||
assert_eq!(
|
||||
crate::heal::outcome::legacy_wire_status("finished", &wire, false)
|
||||
.expect("legacy status")
|
||||
.0,
|
||||
"stopped"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admin_bucket_discovery_complete_empty_cluster_finishes() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
listed_buckets: Mutex::new(Some(Vec::new())),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::Normal);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let task = HealTask::from_request(request, storage);
|
||||
task.execute().await.expect("complete empty namespace is valid");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
assert_eq!(outcome.counters.processed, 0);
|
||||
}
|
||||
|
||||
fn bucket_task(storage: Arc<MockStorage>) -> HealTask {
|
||||
HealTask::from_request(
|
||||
HealRequest::new(
|
||||
@@ -1481,6 +1534,7 @@ struct MockStorage {
|
||||
truncate_without_token: Mutex<bool>,
|
||||
include_object_dir_candidate: Mutex<bool>,
|
||||
listed_buckets: Mutex<Option<Vec<String>>>,
|
||||
bucket_discovery_incomplete: bool,
|
||||
bucket_heal_errors: Mutex<HashMap<String, VecDeque<&'static str>>>,
|
||||
bucket_heal_calls: Mutex<Vec<String>>,
|
||||
block_heal_object: Mutex<bool>,
|
||||
@@ -1980,6 +2034,16 @@ impl HealStorageAPI for MockStorage {
|
||||
Ok(*self.usage_baseline.lock().unwrap())
|
||||
}
|
||||
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
_selector: Option<crate::heal::storage_api::storage::DiskSetSelector>,
|
||||
) -> Result<Vec<BucketInfo>> {
|
||||
if self.bucket_discovery_incomplete {
|
||||
return Err(Error::BucketDiscoveryIncomplete);
|
||||
}
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
let buckets = self
|
||||
.listed_buckets
|
||||
|
||||
@@ -398,6 +398,13 @@ mod tests {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
_selector: Option<crate::heal::storage_api::storage::DiskSetSelector>,
|
||||
) -> Result<Vec<BucketInfo>, Error> {
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>, Error> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
@@ -110,6 +110,13 @@ fn test_heal_task_status_atomic_update() {
|
||||
async fn get_bucket_info(&self, _bucket: &str) -> rustfs_heal::Result<Option<BucketInfo>> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
_selector: Option<storage_api::bug_fixes::DiskSetSelector>,
|
||||
) -> rustfs_heal::Result<Vec<BucketInfo>> {
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<BucketInfo>> {
|
||||
Ok(vec![])
|
||||
}
|
||||
@@ -205,6 +212,13 @@ async fn test_heal_task_transient_object_exists_skip_avoids_recreate() {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
_selector: Option<storage_api::bug_fixes::DiskSetSelector>,
|
||||
) -> rustfs_heal::Result<Vec<BucketInfo>> {
|
||||
self.list_buckets().await
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> rustfs_heal::Result<Vec<BucketInfo>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
|
||||
@@ -79,6 +79,13 @@ impl HealStorageAPI for PausedTraversal {
|
||||
self.inner.get_bucket_info(bucket).await
|
||||
}
|
||||
|
||||
async fn list_buckets_for_admin_heal(
|
||||
&self,
|
||||
selector: Option<storage_api::integration::DiskSetSelector>,
|
||||
) -> Result<Vec<BucketInfo>> {
|
||||
self.inner.list_buckets_for_admin_heal(selector).await
|
||||
}
|
||||
|
||||
async fn list_buckets(&self) -> Result<Vec<BucketInfo>> {
|
||||
self.inner.list_buckets().await
|
||||
}
|
||||
@@ -95,6 +102,10 @@ impl HealStorageAPI for PausedTraversal {
|
||||
self.inner.admit_bucket_incarnation(bucket).await
|
||||
}
|
||||
|
||||
async fn heal_bucket_at_incarnation(&self, bucket: &str, expected: Uuid, opts: &HealOpts) -> Result<HealResultItem> {
|
||||
self.inner.heal_bucket_at_incarnation(bucket, expected, opts).await
|
||||
}
|
||||
|
||||
async fn heal_object_at_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
|
||||
@@ -17,10 +17,11 @@
|
||||
use http::HeaderMap;
|
||||
use rustfs_heal::heal::{
|
||||
manager::{HealConfig, HealManager},
|
||||
outcome::{HealExecutionOutcome, HealTraversalCoverage},
|
||||
storage::{ECStoreHealStorage, HealObjectOptions as ObjectOptions, HealPutObjReader as PutObjReader, HealStorageAPI},
|
||||
task::{HealOptions, HealPriority, HealRequest, HealTaskStatus, HealType},
|
||||
task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType},
|
||||
};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use serial_test::serial;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
@@ -33,7 +34,7 @@ use walkdir::WalkDir;
|
||||
|
||||
mod storage_api;
|
||||
|
||||
use storage_api::integration::{BucketOperations, ECStore, ObjectIO as _, ObjectOperations as _};
|
||||
use storage_api::integration::{BucketOperations, ECStore, ObjectIO as _, ObjectOperations as _, WriteCompletion};
|
||||
|
||||
const HEAL_FORMAT_WAIT_TIMEOUT: Duration = Duration::from_secs(25);
|
||||
const HEAL_FORMAT_WAIT_INTERVAL: Duration = Duration::from_millis(250);
|
||||
@@ -107,16 +108,183 @@ async fn create_test_bucket(ecstore: &Arc<ECStore>, bucket_name: &str) {
|
||||
async fn upload_test_object(ecstore: &Arc<ECStore>, bucket: &str, object: &str, data: &[u8]) {
|
||||
let mut reader = PutObjReader::from_vec(data.to_vec());
|
||||
let object_info = (**ecstore)
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("Failed to upload test object");
|
||||
|
||||
info!("Uploaded test object: {}/{} ({} bytes)", bucket, object, object_info.size);
|
||||
}
|
||||
|
||||
async fn empty_replacement_with_committed_delete(
|
||||
ecstore: &Arc<ECStore>,
|
||||
disk_paths: &[PathBuf],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> (uuid::Uuid, uuid::Uuid) {
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use storage_api::integration::{
|
||||
DeleteOptions, DeleteRecoveryAction, DeleteRecoveryState, DiskAPI as _, DiskSetSelector, StorageAdminApi as _,
|
||||
};
|
||||
assert_eq!(disk_paths.len(), 4, "replacement fixture needs four paths");
|
||||
let incarnation = ecstore
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("admit the survivor bucket generation");
|
||||
let disks = ecstore
|
||||
.disk_set_inventory(DiskSetSelector::new(0, 0))
|
||||
.await
|
||||
.expect("read the live fixture set");
|
||||
assert_eq!(disks.len(), 4);
|
||||
let transaction = uuid::Uuid::new_v4();
|
||||
// Retain a certified survivor decision without the normal delete's
|
||||
// immediate recovery/cleanup consuming the fault input.
|
||||
for disk in &disks[1..] {
|
||||
let disk = disk.as_ref().expect("all three preparation survivors are online");
|
||||
disk.delete_version_recoverable(
|
||||
bucket,
|
||||
object,
|
||||
FileInfo {
|
||||
name: object.to_owned(),
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
delete_recovery_bucket_incarnation: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("prepare the same delete on every survivor");
|
||||
}
|
||||
for disk in &disks[1..] {
|
||||
let disk = disk.as_ref().expect("all three survivors are online");
|
||||
let prepared = disk
|
||||
.inspect_delete_transaction(bucket, object, transaction)
|
||||
.await
|
||||
.expect("inspect prepared survivor decision");
|
||||
assert_eq!(prepared.state, DeleteRecoveryState::NewPrepared);
|
||||
let committed = disk
|
||||
.settle_delete_transaction(bucket, object, transaction, prepared.fingerprint, DeleteRecoveryAction::PublishCommit)
|
||||
.await
|
||||
.expect("publish the retained survivor decision");
|
||||
assert_eq!(committed.state, DeleteRecoveryState::QuorumCommitted);
|
||||
}
|
||||
// Linux disk I/O pins the root inode. Clearing its contents models the
|
||||
// empty target on every platform; renaming it leaves Linux handles on
|
||||
// the retired data and does not simulate a replacement for this test.
|
||||
for entry in std::fs::read_dir(&disk_paths[0]).expect("read target root") {
|
||||
let entry = entry.expect("read target entry");
|
||||
if entry.file_type().expect("target entry type").is_dir() {
|
||||
std::fs::remove_dir_all(entry.path()).expect("clear target directory");
|
||||
} else {
|
||||
std::fs::remove_file(entry.path()).expect("clear target file");
|
||||
}
|
||||
}
|
||||
(incarnation, transaction)
|
||||
}
|
||||
|
||||
mod serial_tests {
|
||||
use super::*;
|
||||
|
||||
async fn bucket_directory_loss_task(cluster: bool, missing: usize, dry_run: bool, source: HealRequestSource) {
|
||||
let temp = tempfile::tempdir().expect("isolated bucket recovery fixture");
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.base_dir(temp.path())
|
||||
.build()
|
||||
.await;
|
||||
let bucket = "task-directory-recovery";
|
||||
env.make_bucket(bucket, false).await;
|
||||
let storage = Arc::new(ECStoreHealStorage::new(env.ecstore.clone()));
|
||||
let expected = storage
|
||||
.admit_bucket_incarnation(bucket)
|
||||
.await
|
||||
.expect("committed bucket generation");
|
||||
for disk in env.disk_paths.iter().rev().take(missing) {
|
||||
fs::remove_dir(disk.join(bucket))
|
||||
.await
|
||||
.expect("remove only an empty bucket directory");
|
||||
}
|
||||
assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).exists()).count(), 4 - missing);
|
||||
let mut request = HealRequest::new(
|
||||
if cluster {
|
||||
HealType::Cluster
|
||||
} else {
|
||||
HealType::Bucket {
|
||||
bucket: bucket.to_owned(),
|
||||
}
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
dry_run,
|
||||
recreate_missing: false,
|
||||
timeout: Some(Duration::from_secs(30)),
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = source;
|
||||
if !cluster && source == HealRequestSource::Admin {
|
||||
request.bucket_incarnation_id = Some(expected);
|
||||
}
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
let result = task.execute().await;
|
||||
if missing == 4 || source != HealRequestSource::Admin {
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"unproven or unbound recovery must fail: missing={missing}, source={source:?}"
|
||||
);
|
||||
} else {
|
||||
result.unwrap_or_else(|error| {
|
||||
panic!(
|
||||
"administrator task must reach generation-bound reconstruction: cluster={cluster}, missing={missing}: {error}"
|
||||
)
|
||||
});
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Complete);
|
||||
}
|
||||
let remaining = if dry_run || missing == 4 || source != HealRequestSource::Admin {
|
||||
4 - missing
|
||||
} else {
|
||||
4
|
||||
};
|
||||
assert_eq!(env.disk_paths.iter().filter(|disk| disk.join(bucket).is_dir()).count(), remaining);
|
||||
assert_eq!(storage.admit_bucket_incarnation(bucket).await.expect("retained metadata"), expected);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_admin_bucket_directory_recovery_from_task_entry() {
|
||||
for cluster in [false, true] {
|
||||
// Two surviving directories satisfy ordinary reads; one does not.
|
||||
for missing in [2, 3] {
|
||||
bucket_directory_loss_task(cluster, missing, false, HealRequestSource::Admin).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_admin_bucket_directory_recovery_dry_run_from_task_entry() {
|
||||
bucket_directory_loss_task(false, 3, true, HealRequestSource::Admin).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_admin_bucket_directory_recovery_rejects_unproven_scope() {
|
||||
bucket_directory_loss_task(false, 4, false, HealRequestSource::Admin).await;
|
||||
bucket_directory_loss_task(false, 3, false, HealRequestSource::Internal).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn test_heal_object_basic() {
|
||||
@@ -533,3 +701,209 @@ mod serial_tests {
|
||||
info!("Direct heal storage API test passed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn replacement_delete_recovery_on_fresh_target() {
|
||||
// Match the debug server's worker stack for the composed Heal future.
|
||||
// Libtest otherwise polls it on its smaller calling-thread stack.
|
||||
const STACK_SIZE: usize = 8 * 1024 * 1024;
|
||||
std::thread::Builder::new()
|
||||
.name("replacement-delete-recovery".to_owned())
|
||||
.stack_size(STACK_SIZE)
|
||||
.spawn(|| {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.worker_threads(4)
|
||||
.thread_stack_size(STACK_SIZE)
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("replacement recovery runtime");
|
||||
runtime.block_on(replacement_delete_recovery_on_fresh_target_case());
|
||||
})
|
||||
.expect("replacement recovery test thread")
|
||||
.join()
|
||||
.expect("replacement recovery test result");
|
||||
}
|
||||
|
||||
async fn replacement_delete_recovery_on_fresh_target_case() {
|
||||
use rustfs_heal::Error;
|
||||
use rustfs_heal::heal::task::HealTask;
|
||||
use rustfs_heal_contracts::heal_channel::HealRequestSource;
|
||||
use storage_api::integration::{
|
||||
DeleteRecoveryState, DiskAPI as _, DiskError, DiskSetSelector, EcstoreStorageError, StorageAdminApi as _,
|
||||
};
|
||||
|
||||
for (source, erasure_set) in [
|
||||
(HealRequestSource::AutoHeal, true),
|
||||
(HealRequestSource::Admin, true),
|
||||
(HealRequestSource::Admin, false),
|
||||
] {
|
||||
let (disk_paths, ecstore, storage) = heal_env().await;
|
||||
let bucket = "replacement-delete-prepass";
|
||||
create_test_bucket(&ecstore, bucket).await;
|
||||
let original = non_inline_test_data();
|
||||
for key in ["live", "deleted"] {
|
||||
upload_test_object(&ecstore, bucket, key, &original).await;
|
||||
wait_for_object_copies(&disk_paths, bucket, key).await;
|
||||
}
|
||||
let (incarnation, transaction) = empty_replacement_with_committed_delete(&ecstore, &disk_paths, bucket, "deleted").await;
|
||||
let updated = vec![17; NON_INLINE_TEST_DATA_SIZE];
|
||||
upload_test_object(&ecstore, bucket, "live", &updated).await;
|
||||
upload_test_object(&ecstore, bucket, "outage-created", &original).await;
|
||||
let (_, error) = storage.heal_format(false).await.expect("format target");
|
||||
assert!(error.is_none(), "{error:?}");
|
||||
assert_eq!(
|
||||
ecstore
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("survivor incarnation remains authoritative"),
|
||||
incarnation
|
||||
);
|
||||
let current_disks = ecstore
|
||||
.disk_set_inventory(DiskSetSelector::new(0, 0))
|
||||
.await
|
||||
.expect("read the formatted live set");
|
||||
assert!(
|
||||
matches!(
|
||||
current_disks[0]
|
||||
.as_ref()
|
||||
.expect("formatted target is attached")
|
||||
.stat_volume(bucket)
|
||||
.await,
|
||||
Err(DiskError::VolumeNotFound)
|
||||
),
|
||||
"the formatted target must still lack its publication bucket"
|
||||
);
|
||||
for disk in ¤t_disks[1..] {
|
||||
let retained = disk
|
||||
.as_ref()
|
||||
.expect("survivor remains attached")
|
||||
.inspect_delete_transaction(bucket, "deleted", transaction)
|
||||
.await
|
||||
.expect("retained committed survivor proof");
|
||||
assert_eq!(retained.state, DeleteRecoveryState::QuorumCommitted);
|
||||
}
|
||||
// A committed survivor journal must be adopted, but publishing its
|
||||
// witness cannot recreate a missing bucket outside lifecycle ownership.
|
||||
let premature = storage
|
||||
.recover_delete_transactions(bucket, None, None, &HealOpts::default())
|
||||
.await;
|
||||
assert!(
|
||||
matches!(&premature, Err(Error::Storage(EcstoreStorageError::FileNotFound))),
|
||||
"{source:?} erasure_set={erasure_set}: {premature:?}"
|
||||
);
|
||||
let heal_type = if erasure_set {
|
||||
HealType::ErasureSet {
|
||||
buckets: vec![bucket.to_owned()],
|
||||
set_disk_id: "pool_0_set_0".to_owned(),
|
||||
}
|
||||
} else {
|
||||
HealType::Bucket {
|
||||
bucket: bucket.to_owned(),
|
||||
}
|
||||
};
|
||||
let mut request = HealRequest::new(
|
||||
heal_type,
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
scan_mode: HealScanMode::Deep,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = source;
|
||||
if !erasure_set {
|
||||
request.bucket_incarnation_id = Some(incarnation);
|
||||
}
|
||||
HealTask::from_request(request, storage.clone())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("{source:?} erasure_set={erasure_set}: {err}"));
|
||||
// Check physical repair before GET can hide missing shards via read repair.
|
||||
for key in ["live", "outage-created"] {
|
||||
assert!(disk_paths[0].join(bucket).join(key).join("xl.meta").exists());
|
||||
assert!(find_part_file(&disk_paths[0].join(bucket).join(key)).is_some());
|
||||
}
|
||||
for (key, expected) in [("live", updated.as_slice()), ("outage-created", original.as_slice())] {
|
||||
let mut reader = ecstore
|
||||
.get_object_reader(bucket, key, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("open rebuilt object");
|
||||
assert_eq!(reader.read_all().await.expect("read rebuilt bytes"), expected);
|
||||
}
|
||||
assert!(matches!(
|
||||
ecstore.get_object_info(bucket, "deleted", &ObjectOptions::default()).await,
|
||||
Err(EcstoreStorageError::ObjectNotFound(_, _))
|
||||
));
|
||||
assert_eq!(
|
||||
ecstore
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("retained incarnation"),
|
||||
incarnation
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn bucket_removal_recovers_delete_proofs_before_removing_empty_volumes() {
|
||||
use rustfs_heal::heal::task::HealTask;
|
||||
use rustfs_heal_contracts::heal_channel::HealRequestSource;
|
||||
use storage_api::integration::{DeleteRecoveryState, DiskAPI as _, DiskSetSelector, StorageAdminApi as _};
|
||||
let (disk_paths, ecstore, storage) = heal_env().await;
|
||||
let bucket = "delete-proof-before-volume-removal";
|
||||
create_test_bucket(&ecstore, bucket).await;
|
||||
upload_test_object(&ecstore, bucket, "deleted", &non_inline_test_data()).await;
|
||||
wait_for_object_copies(&disk_paths, bucket, "deleted").await;
|
||||
let (incarnation, transaction) = empty_replacement_with_committed_delete(&ecstore, &disk_paths, bucket, "deleted").await;
|
||||
let (_, error) = storage.heal_format(false).await.expect("format target");
|
||||
assert!(error.is_none());
|
||||
storage
|
||||
.heal_bucket(
|
||||
bucket,
|
||||
&HealOpts {
|
||||
recreate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("empty target volume");
|
||||
assert!(disk_paths[0].join(bucket).exists());
|
||||
let disks = ecstore
|
||||
.disk_set_inventory(DiskSetSelector::new(0, 0))
|
||||
.await
|
||||
.expect("live removal fixture set");
|
||||
for disk in &disks[1..] {
|
||||
let retained = disk
|
||||
.as_ref()
|
||||
.expect("removal survivor remains attached")
|
||||
.inspect_delete_transaction(bucket, "deleted", transaction)
|
||||
.await
|
||||
.expect("removal must still adopt the retained delete proof");
|
||||
assert_eq!(retained.state, DeleteRecoveryState::QuorumCommitted);
|
||||
}
|
||||
let mut request = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: bucket.to_owned(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: false,
|
||||
remove_corrupted: true,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
request.source = HealRequestSource::Admin;
|
||||
request.bucket_incarnation_id = Some(incarnation);
|
||||
HealTask::from_request(request, storage)
|
||||
.execute()
|
||||
.await
|
||||
.expect("settle journals before deleting empty volumes");
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| !path.join(bucket).exists()),
|
||||
"all empty replicas should be removed after proof settlement"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
pub(crate) mod bug_fixes {
|
||||
pub(crate) use rustfs_ecstore::api::disk::DiskStore;
|
||||
pub(crate) use rustfs_ecstore::api::disk::endpoint::Endpoint;
|
||||
pub(crate) use rustfs_storage_api::BucketInfo;
|
||||
pub(crate) use rustfs_storage_api::{BucketInfo, DiskSetSelector};
|
||||
}
|
||||
|
||||
#[allow(unused_imports)]
|
||||
@@ -21,7 +21,8 @@ pub(crate) mod endpoint_index {
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod integration {
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskAPI, DiskError, DiskOption, DiskStore, Endpoint, RUSTFS_META_BUCKET, ReadOptions, new_disk,
|
||||
DeleteOptions, DeleteRecoveryAction, DeleteRecoveryState, DiskAPI, DiskError, DiskOption, DiskStore, Endpoint,
|
||||
RUSTFS_META_BUCKET, ReadOptions, new_disk,
|
||||
};
|
||||
pub(crate) use rustfs_ecstore::api::error::StorageError as EcstoreStorageError;
|
||||
pub(crate) use rustfs_ecstore::api::object::{
|
||||
|
||||
@@ -12,12 +12,123 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{FederatedAuthorization, VerifiedFederatedIdentity};
|
||||
use super::{FederatedClaims, VerifiedFederatedIdentity};
|
||||
use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
|
||||
use rustfs_policy::policy::{ClaimLookup, get_claim_case_insensitive};
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use tracing::debug;
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
#[path = "test_support.rs"]
|
||||
pub(super) mod test_support;
|
||||
|
||||
/// Read-only authorization result produced by core federation mapping.
|
||||
///
|
||||
/// The claims representation stays inside IAM:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use rustfs_iam::federation::FederatedClaims;
|
||||
/// ```
|
||||
///
|
||||
/// Callers cannot mutate mapped fields:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use rustfs_iam::federation::FederatedAuthorization;
|
||||
///
|
||||
/// fn mutate(authorization: &mut FederatedAuthorization) {
|
||||
/// authorization.groups.clear();
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Callers cannot construct authorization results:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use rustfs_iam::federation::FederatedAuthorization;
|
||||
///
|
||||
/// fn construct(authorization: FederatedAuthorization) -> FederatedAuthorization {
|
||||
/// FederatedAuthorization { ..authorization }
|
||||
/// }
|
||||
/// ```
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FederatedAuthorization {
|
||||
provider_id: String,
|
||||
claims: FederatedClaims,
|
||||
policies: Vec<String>,
|
||||
group_claim_policies: Vec<String>,
|
||||
groups: Vec<String>,
|
||||
roles_claim_key: Option<String>,
|
||||
roles: Vec<String>,
|
||||
}
|
||||
|
||||
impl FederatedAuthorization {
|
||||
pub fn provider_id(&self) -> &str {
|
||||
&self.provider_id
|
||||
}
|
||||
|
||||
pub fn subject(&self) -> &str {
|
||||
&self.claims.sub
|
||||
}
|
||||
|
||||
pub fn email(&self) -> &str {
|
||||
&self.claims.email
|
||||
}
|
||||
|
||||
pub fn username(&self) -> &str {
|
||||
&self.claims.username
|
||||
}
|
||||
|
||||
pub fn session_identity(&self) -> String {
|
||||
self.claims.session_identity()
|
||||
}
|
||||
|
||||
pub fn policies(&self) -> &[String] {
|
||||
&self.policies
|
||||
}
|
||||
|
||||
/// Subset of `policies` derived only from the groups claim (including merged roles claim
|
||||
/// values); these may be ignored when no policy of that name exists. Everything else in
|
||||
/// `policies` must resolve.
|
||||
pub fn group_claim_policies(&self) -> &[String] {
|
||||
&self.group_claim_policies
|
||||
}
|
||||
|
||||
pub fn mapped_iam_groups(&self) -> &[String] {
|
||||
&self.groups
|
||||
}
|
||||
|
||||
pub fn roles_claim_key(&self) -> Option<&str> {
|
||||
self.roles_claim_key.as_deref()
|
||||
}
|
||||
|
||||
pub fn roles(&self) -> &[String] {
|
||||
&self.roles
|
||||
}
|
||||
|
||||
pub fn has_authorization_context(&self) -> bool {
|
||||
!self.policies.is_empty() || !self.groups.is_empty()
|
||||
}
|
||||
|
||||
pub fn oidc_virtual_parent(&self) -> Option<String> {
|
||||
let issuer = self.claims.raw.get("iss")?.as_str()?;
|
||||
let subject = self.claims.sub.as_str();
|
||||
if issuer.is_empty() || subject.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let subject_len = u64::try_from(subject.len()).ok()?;
|
||||
let issuer_len = u64::try_from(issuer.len()).ok()?;
|
||||
let mut source = Vec::with_capacity(23 + subject.len() + issuer.len());
|
||||
source.extend_from_slice(b"openid:");
|
||||
source.extend_from_slice(&subject_len.to_be_bytes());
|
||||
source.extend_from_slice(subject.as_bytes());
|
||||
source.extend_from_slice(&issuer_len.to_be_bytes());
|
||||
source.extend_from_slice(issuer.as_bytes());
|
||||
let digest = HashAlgorithm::SHA256.hash_encode(&source);
|
||||
Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(digest.as_ref()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct FederatedAuthorizationRule {
|
||||
provider_id: String,
|
||||
@@ -308,8 +419,136 @@ fn claim_value_type(value: &serde_json::Value) -> &'static str {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::federation::{FederatedClaims, FederatedProviderRef, VerifiedFederatedIdentity};
|
||||
use serde_json::json;
|
||||
use crate::federation::{FederatedAuthorizationFixture, FederatedClaims, FederatedProviderRef, VerifiedFederatedIdentity};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
fn authorization(policies: Vec<String>, groups: Vec<String>) -> FederatedAuthorization {
|
||||
FederatedAuthorizationFixture {
|
||||
provider_id: "standard_oidc".to_string(),
|
||||
sub: "subject".to_string(),
|
||||
attributes: HashMap::from([("iss".to_string(), Value::String("https://idp.example.test".to_string()))]),
|
||||
policies,
|
||||
mapped_iam_groups: groups,
|
||||
..Default::default()
|
||||
}
|
||||
.into_authorization()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authorization_context_accepts_policy_or_group() {
|
||||
assert!(!authorization(Vec::new(), Vec::new()).has_authorization_context());
|
||||
assert!(authorization(vec!["consoleAdmin".to_string()], Vec::new()).has_authorization_context());
|
||||
assert!(authorization(Vec::new(), vec!["RustFS.ConsoleAdmin".to_string()]).has_authorization_context());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_virtual_parent_is_issuer_scoped() {
|
||||
let first = authorization(Vec::new(), Vec::new());
|
||||
let mut second = first.clone();
|
||||
second
|
||||
.claims
|
||||
.raw
|
||||
.insert("iss".to_string(), Value::String("https://other-idp.example.test".to_string()));
|
||||
|
||||
assert_eq!(
|
||||
first.oidc_virtual_parent().as_deref(),
|
||||
Some("HwDfWftzOy4jiuS3WjKytC_Sg_A2hKhrRAFtBDhoBr0")
|
||||
);
|
||||
assert!(!rustfs_policy::auth::contains_reserved_chars(
|
||||
first.oidc_virtual_parent().as_deref().expect("virtual parent")
|
||||
));
|
||||
assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_virtual_parent_length_delimits_identity_parts() {
|
||||
let mut first = authorization(Vec::new(), Vec::new());
|
||||
first.claims.sub = "subject".to_string();
|
||||
first.claims.raw.insert(
|
||||
"iss".to_string(),
|
||||
Value::String("https://issuer.example/path:https://other.example".to_string()),
|
||||
);
|
||||
let mut second = authorization(Vec::new(), Vec::new());
|
||||
second.claims.sub = "subject:https://issuer.example/path".to_string();
|
||||
second
|
||||
.claims
|
||||
.raw
|
||||
.insert("iss".to_string(), Value::String("https://other.example".to_string()));
|
||||
|
||||
assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_virtual_parent_requires_verified_identity_parts() {
|
||||
let mut missing_issuer = authorization(Vec::new(), Vec::new());
|
||||
missing_issuer.claims.raw.clear();
|
||||
let mut missing_subject = authorization(Vec::new(), Vec::new());
|
||||
missing_subject.claims.sub.clear();
|
||||
|
||||
assert!(missing_issuer.oidc_virtual_parent().is_none());
|
||||
assert!(missing_subject.oidc_virtual_parent().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_virtual_parent_preserves_exact_subject() {
|
||||
let plain = authorization(Vec::new(), Vec::new());
|
||||
let mut padded = plain.clone();
|
||||
padded.claims.sub = format!(" {} ", plain.claims.sub);
|
||||
|
||||
assert_ne!(plain.oidc_virtual_parent(), padded.oidc_virtual_parent());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authorization_accessors_preserve_exact_values_and_distinct_groups() {
|
||||
let attributes = HashMap::from([("custom".to_string(), json!(["value", null, 7]))]);
|
||||
let authorization = FederatedAuthorizationFixture {
|
||||
provider_id: " Corp Provider ".to_string(),
|
||||
sub: " subject ".to_string(),
|
||||
email: " email@example.test ".to_string(),
|
||||
username: " username ".to_string(),
|
||||
source_groups: vec!["different-source-group".to_string()],
|
||||
attributes: attributes.clone(),
|
||||
policies: vec![" policy-b ".to_string(), "policy-a".to_string(), " policy-b ".to_string()],
|
||||
group_claim_policies: vec![" policy-b ".to_string(), " policy-b ".to_string()],
|
||||
mapped_iam_groups: vec!["mapped-a".to_string(), "mapped-b".to_string()],
|
||||
roles_claim_key: Some(" Roles ".to_string()),
|
||||
roles: vec!["role-b".to_string(), " role-a ".to_string(), "role-b".to_string()],
|
||||
}
|
||||
.into_authorization();
|
||||
|
||||
assert_eq!(authorization.provider_id(), " Corp Provider ");
|
||||
assert_eq!(authorization.subject(), " subject ");
|
||||
assert_eq!(authorization.email(), " email@example.test ");
|
||||
assert_eq!(authorization.username(), " username ");
|
||||
assert_eq!(authorization.session_identity(), " username ");
|
||||
assert_eq!(authorization.policies(), [" policy-b ", "policy-a", " policy-b "]);
|
||||
assert_eq!(authorization.group_claim_policies(), [" policy-b ", " policy-b "]);
|
||||
assert_eq!(authorization.claims.groups, ["different-source-group"]);
|
||||
assert_eq!(authorization.claims.raw, attributes);
|
||||
assert_eq!(authorization.mapped_iam_groups(), ["mapped-a", "mapped-b"]);
|
||||
assert_eq!(authorization.roles_claim_key(), Some(" Roles "));
|
||||
assert_eq!(authorization.roles(), ["role-b", " role-a ", "role-b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_authorization_fixture_has_no_identity_or_mapping() {
|
||||
let authorization = FederatedAuthorizationFixture::default().into_authorization();
|
||||
|
||||
assert_eq!(authorization.provider_id(), "");
|
||||
assert_eq!(authorization.subject(), "");
|
||||
assert_eq!(authorization.email(), "");
|
||||
assert_eq!(authorization.username(), "");
|
||||
assert_eq!(authorization.session_identity(), "oidc-user-unknown");
|
||||
assert!(authorization.claims.groups.is_empty());
|
||||
assert!(authorization.claims.raw.is_empty());
|
||||
assert!(authorization.policies().is_empty());
|
||||
assert!(authorization.group_claim_policies().is_empty());
|
||||
assert!(authorization.mapped_iam_groups().is_empty());
|
||||
assert!(authorization.roles_claim_key().is_none());
|
||||
assert!(authorization.roles().is_empty());
|
||||
assert!(!authorization.has_authorization_context());
|
||||
assert!(authorization.oidc_virtual_parent().is_none());
|
||||
}
|
||||
|
||||
fn mapper(rule: FederatedAuthorizationRule) -> CoreFederatedAuthorizationMapper {
|
||||
CoreFederatedAuthorizationMapper::new(FederatedAuthorizationRules::new([rule]))
|
||||
|
||||
@@ -24,11 +24,13 @@ pub use error::{FederatedSessionBindingError, FederationError, Result};
|
||||
pub(crate) use mapper::FederatedAuthorizationRule;
|
||||
#[cfg(test)]
|
||||
pub(crate) use mapper::FederatedAuthorizationRuleRef;
|
||||
pub use mapper::{CoreFederatedAuthorizationMapper, FederatedAuthorizationRules};
|
||||
pub(crate) use model::OpaqueLogoutContinuation;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub use mapper::test_support::FederatedAuthorizationFixture;
|
||||
pub use mapper::{CoreFederatedAuthorizationMapper, FederatedAuthorization, FederatedAuthorizationRules};
|
||||
pub(crate) use model::{FederatedClaims, OpaqueLogoutContinuation};
|
||||
pub use model::{
|
||||
FederatedAuthorization, FederatedClaims, FederatedLoginSession, FederatedProviderRef, FederatedSession,
|
||||
FederatedSessionTransaction, OIDC_VIRTUAL_PARENT_CLAIM, VerifiedFederatedIdentity,
|
||||
FederatedLoginSession, FederatedProviderRef, FederatedSession, FederatedSessionTransaction, OIDC_VIRTUAL_PARENT_CLAIM,
|
||||
VerifiedFederatedIdentity,
|
||||
};
|
||||
pub(crate) use provider::{FederatedProviderQuery, StandardOidcAuthentication, VerifiedFederatedCodeExchange};
|
||||
pub use provider::{FederatedProviderView, FederatedRedirectPolicy};
|
||||
|
||||
@@ -13,23 +13,24 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_credentials::Credentials;
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use serde_json::Value;
|
||||
use std::{collections::HashMap, fmt};
|
||||
|
||||
use super::FederatedAuthorization;
|
||||
|
||||
pub const OIDC_VIRTUAL_PARENT_CLAIM: &str = "x-rustfs-internal-oidc-parent";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FederatedClaims {
|
||||
pub sub: String,
|
||||
pub email: String,
|
||||
pub username: String,
|
||||
pub groups: Vec<String>,
|
||||
pub raw: HashMap<String, Value>,
|
||||
pub(crate) struct FederatedClaims {
|
||||
pub(crate) sub: String,
|
||||
pub(crate) email: String,
|
||||
pub(crate) username: String,
|
||||
pub(crate) groups: Vec<String>,
|
||||
pub(crate) raw: HashMap<String, Value>,
|
||||
}
|
||||
|
||||
impl FederatedClaims {
|
||||
pub fn session_identity(&self) -> String {
|
||||
pub(crate) fn session_identity(&self) -> String {
|
||||
if !self.username.is_empty() {
|
||||
self.username.clone()
|
||||
} else if !self.email.is_empty() {
|
||||
@@ -125,45 +126,6 @@ impl VerifiedFederatedIdentity {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FederatedAuthorization {
|
||||
pub provider_id: String,
|
||||
pub claims: FederatedClaims,
|
||||
pub policies: Vec<String>,
|
||||
/// Subset of `policies` derived only from the groups claim (including merged roles claim
|
||||
/// values); these may be ignored when no policy of that name exists. Everything else in
|
||||
/// `policies` must resolve.
|
||||
pub group_claim_policies: Vec<String>,
|
||||
pub groups: Vec<String>,
|
||||
pub roles_claim_key: Option<String>,
|
||||
pub roles: Vec<String>,
|
||||
}
|
||||
|
||||
impl FederatedAuthorization {
|
||||
pub fn has_authorization_context(&self) -> bool {
|
||||
!self.policies.is_empty() || !self.groups.is_empty()
|
||||
}
|
||||
|
||||
pub fn oidc_virtual_parent(&self) -> Option<String> {
|
||||
let issuer = self.claims.raw.get("iss")?.as_str()?;
|
||||
let subject = self.claims.sub.as_str();
|
||||
if issuer.is_empty() || subject.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let subject_len = u64::try_from(subject.len()).ok()?;
|
||||
let issuer_len = u64::try_from(issuer.len()).ok()?;
|
||||
let mut source = Vec::with_capacity(23 + subject.len() + issuer.len());
|
||||
source.extend_from_slice(b"openid:");
|
||||
source.extend_from_slice(&subject_len.to_be_bytes());
|
||||
source.extend_from_slice(subject.as_bytes());
|
||||
source.extend_from_slice(&issuer_len.to_be_bytes());
|
||||
source.extend_from_slice(issuer.as_bytes());
|
||||
let digest = HashAlgorithm::SHA256.hash_encode(&source);
|
||||
Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(digest.as_ref()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Carries the provider-bound ID token between code exchange and logout token creation.
|
||||
pub(crate) struct OpaqueLogoutContinuation {
|
||||
provider: FederatedProviderRef,
|
||||
@@ -224,21 +186,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn authorization(policies: Vec<String>, groups: Vec<String>) -> FederatedAuthorization {
|
||||
FederatedAuthorization {
|
||||
provider_id: "standard_oidc".to_string(),
|
||||
claims: FederatedClaims {
|
||||
raw: HashMap::from([("iss".to_string(), Value::String("https://idp.example.test".to_string()))]),
|
||||
..claims("", "", "subject")
|
||||
},
|
||||
policies,
|
||||
group_claim_policies: Vec::new(),
|
||||
groups,
|
||||
roles_claim_key: None,
|
||||
roles: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_identity_preserves_existing_fallback_order() {
|
||||
assert_eq!(claims("john", "john@example.com", "sub-1").session_identity(), "john");
|
||||
@@ -247,70 +194,6 @@ mod tests {
|
||||
assert_eq!(claims("", "", "").session_identity(), "oidc-user-unknown");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authorization_context_accepts_policy_or_group() {
|
||||
assert!(!authorization(Vec::new(), Vec::new()).has_authorization_context());
|
||||
assert!(authorization(vec!["consoleAdmin".to_string()], Vec::new()).has_authorization_context());
|
||||
assert!(authorization(Vec::new(), vec!["RustFS.ConsoleAdmin".to_string()]).has_authorization_context());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_virtual_parent_is_issuer_scoped() {
|
||||
let first = authorization(Vec::new(), Vec::new());
|
||||
let mut second = first.clone();
|
||||
second
|
||||
.claims
|
||||
.raw
|
||||
.insert("iss".to_string(), Value::String("https://other-idp.example.test".to_string()));
|
||||
|
||||
assert_eq!(
|
||||
first.oidc_virtual_parent().as_deref(),
|
||||
Some("HwDfWftzOy4jiuS3WjKytC_Sg_A2hKhrRAFtBDhoBr0")
|
||||
);
|
||||
assert!(!rustfs_policy::auth::contains_reserved_chars(
|
||||
first.oidc_virtual_parent().as_deref().expect("virtual parent")
|
||||
));
|
||||
assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_virtual_parent_length_delimits_identity_parts() {
|
||||
let mut first = authorization(Vec::new(), Vec::new());
|
||||
first.claims.sub = "subject".to_string();
|
||||
first.claims.raw.insert(
|
||||
"iss".to_string(),
|
||||
Value::String("https://issuer.example/path:https://other.example".to_string()),
|
||||
);
|
||||
let mut second = authorization(Vec::new(), Vec::new());
|
||||
second.claims.sub = "subject:https://issuer.example/path".to_string();
|
||||
second
|
||||
.claims
|
||||
.raw
|
||||
.insert("iss".to_string(), Value::String("https://other.example".to_string()));
|
||||
|
||||
assert_ne!(first.oidc_virtual_parent(), second.oidc_virtual_parent());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_virtual_parent_requires_verified_identity_parts() {
|
||||
let mut missing_issuer = authorization(Vec::new(), Vec::new());
|
||||
missing_issuer.claims.raw.clear();
|
||||
let mut missing_subject = authorization(Vec::new(), Vec::new());
|
||||
missing_subject.claims.sub.clear();
|
||||
|
||||
assert!(missing_issuer.oidc_virtual_parent().is_none());
|
||||
assert!(missing_subject.oidc_virtual_parent().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oidc_virtual_parent_preserves_exact_subject() {
|
||||
let plain = authorization(Vec::new(), Vec::new());
|
||||
let mut padded = plain.clone();
|
||||
padded.claims.sub = format!(" {} ", plain.claims.sub);
|
||||
|
||||
assert_ne!(plain.oidc_virtual_parent(), padded.oidc_virtual_parent());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verified_identity_preserves_exact_values_and_distinct_groups() {
|
||||
let attributes = HashMap::from([
|
||||
@@ -341,24 +224,6 @@ mod tests {
|
||||
assert!(!debug.contains(" username "));
|
||||
assert!(!debug.contains(" source-b "));
|
||||
assert!(!debug.contains("identity-attribute-poison"));
|
||||
|
||||
let authorization = FederatedAuthorization {
|
||||
provider_id: " Corp Provider ".to_string(),
|
||||
claims: FederatedClaims {
|
||||
sub: "subject".to_string(),
|
||||
email: String::new(),
|
||||
username: String::new(),
|
||||
groups: vec!["different-source-group".to_string()],
|
||||
raw: HashMap::new(),
|
||||
},
|
||||
policies: Vec::new(),
|
||||
group_claim_policies: Vec::new(),
|
||||
groups: vec!["mapped-a".to_string(), "mapped-b".to_string()],
|
||||
roles_claim_key: None,
|
||||
roles: Vec::new(),
|
||||
};
|
||||
assert_eq!(authorization.claims.groups, ["different-source-group"]);
|
||||
assert_eq!(authorization.groups, ["mapped-a", "mapped-b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{FederatedAuthorization, FederatedClaims};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Exact authorization inputs for tests that exercise the session binding boundary.
|
||||
#[derive(Default)]
|
||||
pub struct FederatedAuthorizationFixture {
|
||||
pub provider_id: String,
|
||||
pub sub: String,
|
||||
pub email: String,
|
||||
pub username: String,
|
||||
pub source_groups: Vec<String>,
|
||||
pub attributes: HashMap<String, Value>,
|
||||
pub policies: Vec<String>,
|
||||
pub group_claim_policies: Vec<String>,
|
||||
pub mapped_iam_groups: Vec<String>,
|
||||
pub roles_claim_key: Option<String>,
|
||||
pub roles: Vec<String>,
|
||||
}
|
||||
|
||||
impl FederatedAuthorizationFixture {
|
||||
/// Preserves supplied values without authentication, validation, or policy mapping.
|
||||
pub fn into_authorization(self) -> FederatedAuthorization {
|
||||
FederatedAuthorization {
|
||||
provider_id: self.provider_id,
|
||||
claims: FederatedClaims {
|
||||
sub: self.sub,
|
||||
email: self.email,
|
||||
username: self.username,
|
||||
groups: self.source_groups,
|
||||
raw: self.attributes,
|
||||
},
|
||||
policies: self.policies,
|
||||
group_claim_policies: self.group_claim_policies,
|
||||
groups: self.mapped_iam_groups,
|
||||
roles_claim_key: self.roles_claim_key,
|
||||
roles: self.roles,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ use super::{
|
||||
FederatedProviderView, FederatedRedirectPolicy, FederatedSession, FederatedSessionBinding, FederatedSessionTransaction,
|
||||
FederationError, OpaqueLogoutContinuation, Result, StandardOidcAuthentication,
|
||||
};
|
||||
use crate::{EVENT_OIDC_DIAGNOSTICS, LOG_COMPONENT_IAM, LOG_SUBSYSTEM_OIDC};
|
||||
use std::sync::Arc;
|
||||
|
||||
const DEFAULT_OIDC_PROVIDER_ID: &str = "default";
|
||||
@@ -133,22 +134,28 @@ impl FederatedIdentityService {
|
||||
let authorization = self.verify_identity(jwt).await?;
|
||||
if !authorization.has_authorization_context() {
|
||||
tracing::warn!(
|
||||
provider_id = %authorization.provider_id,
|
||||
username = %authorization.claims.username,
|
||||
sub = %authorization.claims.sub,
|
||||
policy_count = authorization.policies.len(),
|
||||
group_count = authorization.groups.len(),
|
||||
event = EVENT_OIDC_DIAGNOSTICS,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_OIDC,
|
||||
provider_id = %authorization.provider_id(),
|
||||
username = %authorization.username(),
|
||||
sub = %authorization.subject(),
|
||||
policy_count = authorization.policies().len(),
|
||||
group_count = authorization.mapped_iam_groups().len(),
|
||||
"AssumeRoleWithWebIdentity has no mapped policies or groups"
|
||||
);
|
||||
return Err(FederationError::NoAuthorizationContext);
|
||||
}
|
||||
tracing::debug!(
|
||||
provider_id = %authorization.provider_id,
|
||||
username = %authorization.claims.username,
|
||||
policy_count = authorization.policies.len(),
|
||||
group_count = authorization.groups.len(),
|
||||
policies = ?authorization.policies,
|
||||
groups = ?authorization.groups,
|
||||
event = EVENT_OIDC_DIAGNOSTICS,
|
||||
component = LOG_COMPONENT_IAM,
|
||||
subsystem = LOG_SUBSYSTEM_OIDC,
|
||||
provider_id = %authorization.provider_id(),
|
||||
username = %authorization.username(),
|
||||
policy_count = authorization.policies().len(),
|
||||
group_count = authorization.mapped_iam_groups().len(),
|
||||
policies = ?authorization.policies(),
|
||||
groups = ?authorization.mapped_iam_groups(),
|
||||
"AssumeRoleWithWebIdentity mapped OIDC policies and groups"
|
||||
);
|
||||
|
||||
@@ -379,19 +386,19 @@ mod tests {
|
||||
) -> core::result::Result<Credentials, FederatedSessionBindingError> {
|
||||
self.events.lock().expect("event log should not be poisoned").push("bind");
|
||||
self.transactions.lock().expect("transactions should not be poisoned").push((
|
||||
transaction.authorization.provider_id.clone(),
|
||||
transaction.authorization.provider_id().to_string(),
|
||||
transaction.duration_seconds,
|
||||
transaction.session_policy.clone(),
|
||||
));
|
||||
self.mapped_policies
|
||||
.lock()
|
||||
.expect("mapped policies should not be poisoned")
|
||||
.push(transaction.authorization.policies.clone());
|
||||
.push(transaction.authorization.policies().to_vec());
|
||||
if self.fail {
|
||||
return Err(FederatedSessionBindingError::Internal("binding failed".to_string()));
|
||||
}
|
||||
Ok(Credentials {
|
||||
access_key: transaction.authorization.claims.session_identity(),
|
||||
access_key: transaction.authorization.session_identity(),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
@@ -487,7 +494,7 @@ mod tests {
|
||||
.await
|
||||
.expect("callback flow should complete");
|
||||
assert_eq!(login.session.credentials.access_key, "user");
|
||||
assert_eq!(login.session.authorization.provider_id, "corp");
|
||||
assert_eq!(login.session.authorization.provider_id(), "corp");
|
||||
assert_eq!(login.redirect_after.as_deref(), Some("/browser"));
|
||||
assert_eq!(login.logout_token, "logout-token");
|
||||
assert_eq!(
|
||||
@@ -501,7 +508,7 @@ mod tests {
|
||||
.await
|
||||
.expect("web identity flow should complete");
|
||||
assert_eq!(web_identity.credentials.access_key, "user");
|
||||
assert_eq!(web_identity.authorization.provider_id, "partner");
|
||||
assert_eq!(web_identity.authorization.provider_id(), "partner");
|
||||
assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]);
|
||||
assert_eq!(
|
||||
binding
|
||||
@@ -530,8 +537,8 @@ mod tests {
|
||||
.complete_authorization_code("state", "code", "https://console.example/callback", 3600, binding.as_ref())
|
||||
.await
|
||||
.expect("standard callback flow should complete");
|
||||
assert_eq!(login.session.authorization.provider_id, "corp");
|
||||
assert_eq!(login.session.authorization.policies, ["readwrite"]);
|
||||
assert_eq!(login.session.authorization.provider_id(), "corp");
|
||||
assert_eq!(login.session.authorization.policies(), ["readwrite"]);
|
||||
assert_eq!(
|
||||
events.lock().expect("event log should not be poisoned").as_slice(),
|
||||
["exchange", "bind", "logout"]
|
||||
@@ -542,8 +549,8 @@ mod tests {
|
||||
.assume_role_with_web_identity("jwt", 7200, Some("session-policy".to_string()), binding.as_ref())
|
||||
.await
|
||||
.expect("standard web identity flow should complete");
|
||||
assert_eq!(web_identity.authorization.provider_id, "partner");
|
||||
assert_eq!(web_identity.authorization.policies, ["readwrite"]);
|
||||
assert_eq!(web_identity.authorization.provider_id(), "partner");
|
||||
assert_eq!(web_identity.authorization.policies(), ["readwrite"]);
|
||||
assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]);
|
||||
assert_eq!(
|
||||
binding
|
||||
@@ -586,8 +593,8 @@ mod tests {
|
||||
.await
|
||||
.expect("a mapped group is an authorization context");
|
||||
|
||||
assert_eq!(session.authorization.policies, ["developers"]);
|
||||
assert_eq!(session.authorization.groups, ["developers"]);
|
||||
assert_eq!(session.authorization.policies(), ["developers"]);
|
||||
assert_eq!(session.authorization.mapped_iam_groups(), ["developers"]);
|
||||
assert_eq!(events.lock().expect("event log should not be poisoned").as_slice(), ["verify", "bind"]);
|
||||
}
|
||||
|
||||
|
||||
@@ -177,9 +177,9 @@ async fn web_identity_recovers_transport_failure_and_preserves_verification() {
|
||||
.await
|
||||
.expect("verify recovered workload token");
|
||||
let authorization = mapper.map(identity);
|
||||
assert_eq!(authorization.provider_id, "workload");
|
||||
assert_eq!(authorization.claims.sub, "recovered-user");
|
||||
assert_eq!(authorization.policies, vec!["team/readonly"]);
|
||||
assert_eq!(authorization.provider_id(), "workload");
|
||||
assert_eq!(authorization.subject(), "recovered-user");
|
||||
assert_eq!(authorization.policies(), ["team/readonly"]);
|
||||
assert!(
|
||||
sys.authorize_url("workload", "https://console.example.com/callback", None)
|
||||
.await
|
||||
|
||||
@@ -371,12 +371,12 @@ async fn assert_complete_console_login(initially_unavailable: bool) {
|
||||
.await
|
||||
.unwrap_or_else(|err| panic!("hidden={hidden}, explicit={explicit}: {err}"));
|
||||
let (identity, redirect_after, _) = exchange.into_parts();
|
||||
assert_eq!(identity.source_groups(), ["readwrite"]);
|
||||
let authorization = mapper.map(identity);
|
||||
assert_eq!(authorization.provider_id, "console");
|
||||
assert_eq!(authorization.claims.sub, "existing-user");
|
||||
assert_eq!(authorization.claims.email, "user@example.com");
|
||||
assert_eq!(authorization.claims.groups, vec!["readwrite"]);
|
||||
assert_eq!(authorization.policies, vec!["team/readwrite"]);
|
||||
assert_eq!(authorization.provider_id(), "console");
|
||||
assert_eq!(authorization.subject(), "existing-user");
|
||||
assert_eq!(authorization.email(), "user@example.com");
|
||||
assert_eq!(authorization.policies(), ["team/readwrite"]);
|
||||
assert_eq!(redirect_after.as_deref(), Some("/buckets"));
|
||||
assert!(matches!(sys.exchange_code(&state, "test-authorization-code", redirect).await,
|
||||
Err(error) if error == "invalid or expired OIDC state"));
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user